2024-11-17 15:36:35,965 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-17 15:36:35,976 main DEBUG Took 0.009027 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-17 15:36:35,976 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-17 15:36:35,976 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-17 15:36:35,977 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-17 15:36:35,978 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 15:36:35,985 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-17 15:36:35,996 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 15:36:35,997 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 15:36:35,998 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 15:36:35,998 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 15:36:35,999 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 15:36:35,999 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 15:36:36,000 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 15:36:36,000 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 15:36:36,001 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 15:36:36,001 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 15:36:36,002 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 15:36:36,002 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 15:36:36,002 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 15:36:36,002 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 15:36:36,003 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 15:36:36,003 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 15:36:36,004 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 15:36:36,004 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 15:36:36,004 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 15:36:36,005 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 15:36:36,005 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 15:36:36,006 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 15:36:36,006 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 15:36:36,006 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 15:36:36,007 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 15:36:36,007 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-17 15:36:36,008 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 15:36:36,009 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-17 15:36:36,011 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-17 15:36:36,011 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-17 15:36:36,012 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-17 15:36:36,013 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-17 15:36:36,021 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-17 15:36:36,023 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-17 15:36:36,025 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-17 15:36:36,025 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-17 15:36:36,026 main DEBUG createAppenders(={Console}) 2024-11-17 15:36:36,026 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-11-17 15:36:36,027 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-17 15:36:36,027 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-11-17 15:36:36,028 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-17 15:36:36,028 main DEBUG OutputStream closed 2024-11-17 15:36:36,029 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-17 15:36:36,029 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-17 15:36:36,029 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-11-17 15:36:36,102 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-17 15:36:36,104 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-17 15:36:36,104 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-17 15:36:36,105 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-17 15:36:36,106 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-17 15:36:36,106 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-17 15:36:36,106 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-17 15:36:36,107 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-17 15:36:36,107 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-17 15:36:36,107 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-17 15:36:36,108 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-17 15:36:36,108 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-17 15:36:36,108 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-17 15:36:36,109 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-17 15:36:36,109 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-17 15:36:36,109 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-17 15:36:36,109 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-17 15:36:36,110 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-17 15:36:36,112 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-17 15:36:36,113 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-11-17 15:36:36,113 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-17 15:36:36,113 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-11-17T15:36:36,398 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6ce30f36-a42e-0287-cfce-b614a9c6228b 2024-11-17 15:36:36,401 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-17 15:36:36,401 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-17T15:36:36,410 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithNoInMemCompaction timeout: 13 mins 2024-11-17T15:36:36,429 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-17T15:36:36,450 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6ce30f36-a42e-0287-cfce-b614a9c6228b/cluster_3bb18fcf-0ee3-9799-c677-a8d421ff97cf, deleteOnExit=true 2024-11-17T15:36:36,450 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-17T15:36:36,451 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6ce30f36-a42e-0287-cfce-b614a9c6228b/test.cache.data in system properties and HBase conf 2024-11-17T15:36:36,452 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6ce30f36-a42e-0287-cfce-b614a9c6228b/hadoop.tmp.dir in system properties and HBase conf 2024-11-17T15:36:36,453 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6ce30f36-a42e-0287-cfce-b614a9c6228b/hadoop.log.dir in system properties and HBase conf 2024-11-17T15:36:36,453 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6ce30f36-a42e-0287-cfce-b614a9c6228b/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-17T15:36:36,454 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6ce30f36-a42e-0287-cfce-b614a9c6228b/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-17T15:36:36,454 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-17T15:36:36,546 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-17T15:36:36,636 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-17T15:36:36,640 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6ce30f36-a42e-0287-cfce-b614a9c6228b/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-17T15:36:36,641 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6ce30f36-a42e-0287-cfce-b614a9c6228b/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-17T15:36:36,642 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6ce30f36-a42e-0287-cfce-b614a9c6228b/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-17T15:36:36,642 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6ce30f36-a42e-0287-cfce-b614a9c6228b/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-17T15:36:36,643 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6ce30f36-a42e-0287-cfce-b614a9c6228b/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-17T15:36:36,643 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6ce30f36-a42e-0287-cfce-b614a9c6228b/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-17T15:36:36,644 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6ce30f36-a42e-0287-cfce-b614a9c6228b/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-17T15:36:36,645 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6ce30f36-a42e-0287-cfce-b614a9c6228b/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-17T15:36:36,645 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6ce30f36-a42e-0287-cfce-b614a9c6228b/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-17T15:36:36,646 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6ce30f36-a42e-0287-cfce-b614a9c6228b/nfs.dump.dir in system properties and HBase conf 2024-11-17T15:36:36,646 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6ce30f36-a42e-0287-cfce-b614a9c6228b/java.io.tmpdir in system properties and HBase conf 2024-11-17T15:36:36,647 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6ce30f36-a42e-0287-cfce-b614a9c6228b/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-17T15:36:36,647 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6ce30f36-a42e-0287-cfce-b614a9c6228b/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-17T15:36:36,648 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6ce30f36-a42e-0287-cfce-b614a9c6228b/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-17T15:36:37,466 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-17T15:36:37,542 INFO [Time-limited test {}] log.Log(170): Logging initialized @2267ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-17T15:36:37,616 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-17T15:36:37,676 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-17T15:36:37,696 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-17T15:36:37,696 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-17T15:36:37,697 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-17T15:36:37,709 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-17T15:36:37,712 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6ce30f36-a42e-0287-cfce-b614a9c6228b/hadoop.log.dir/,AVAILABLE} 2024-11-17T15:36:37,713 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-17T15:36:37,900 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4ded99d3{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6ce30f36-a42e-0287-cfce-b614a9c6228b/java.io.tmpdir/jetty-localhost-39089-hadoop-hdfs-3_4_1-tests_jar-_-any-8037813053913888108/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-17T15:36:37,907 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:39089} 2024-11-17T15:36:37,907 INFO [Time-limited test {}] server.Server(415): Started @2633ms 2024-11-17T15:36:38,282 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-17T15:36:38,288 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-17T15:36:38,289 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-17T15:36:38,289 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-17T15:36:38,290 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-17T15:36:38,291 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6ce30f36-a42e-0287-cfce-b614a9c6228b/hadoop.log.dir/,AVAILABLE} 2024-11-17T15:36:38,291 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-17T15:36:38,413 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4269bd31{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6ce30f36-a42e-0287-cfce-b614a9c6228b/java.io.tmpdir/jetty-localhost-39981-hadoop-hdfs-3_4_1-tests_jar-_-any-8835718383973931129/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-17T15:36:38,414 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:39981} 2024-11-17T15:36:38,415 INFO [Time-limited test {}] server.Server(415): Started @3141ms 2024-11-17T15:36:38,469 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-17T15:36:38,950 WARN [Thread-71 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6ce30f36-a42e-0287-cfce-b614a9c6228b/cluster_3bb18fcf-0ee3-9799-c677-a8d421ff97cf/data/data1/current/BP-976341801-172.17.0.2-1731857797236/current, will proceed with Du for space computation calculation, 2024-11-17T15:36:38,950 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6ce30f36-a42e-0287-cfce-b614a9c6228b/cluster_3bb18fcf-0ee3-9799-c677-a8d421ff97cf/data/data2/current/BP-976341801-172.17.0.2-1731857797236/current, will proceed with Du for space computation calculation, 2024-11-17T15:36:38,991 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-17T15:36:39,044 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x7258a2cee55161e3 with lease ID 0x6681d4cb1afc61ea: Processing first storage report for DS-f414f29e-7b04-4193-b099-6d2b4232d69f from datanode DatanodeRegistration(127.0.0.1:46871, datanodeUuid=d17f3f9e-828e-4c49-97dc-417781b49415, infoPort=39733, infoSecurePort=0, ipcPort=39609, storageInfo=lv=-57;cid=testClusterID;nsid=1416426980;c=1731857797236) 2024-11-17T15:36:39,045 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x7258a2cee55161e3 with lease ID 0x6681d4cb1afc61ea: from storage DS-f414f29e-7b04-4193-b099-6d2b4232d69f node DatanodeRegistration(127.0.0.1:46871, datanodeUuid=d17f3f9e-828e-4c49-97dc-417781b49415, infoPort=39733, infoSecurePort=0, ipcPort=39609, storageInfo=lv=-57;cid=testClusterID;nsid=1416426980;c=1731857797236), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-17T15:36:39,045 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x7258a2cee55161e3 with lease ID 0x6681d4cb1afc61ea: Processing first storage report for DS-0ef98de3-5cfc-4392-8ba8-4a8c90c07c38 from datanode DatanodeRegistration(127.0.0.1:46871, datanodeUuid=d17f3f9e-828e-4c49-97dc-417781b49415, infoPort=39733, infoSecurePort=0, ipcPort=39609, storageInfo=lv=-57;cid=testClusterID;nsid=1416426980;c=1731857797236) 2024-11-17T15:36:39,045 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x7258a2cee55161e3 with lease ID 0x6681d4cb1afc61ea: from storage DS-0ef98de3-5cfc-4392-8ba8-4a8c90c07c38 node DatanodeRegistration(127.0.0.1:46871, datanodeUuid=d17f3f9e-828e-4c49-97dc-417781b49415, infoPort=39733, infoSecurePort=0, ipcPort=39609, storageInfo=lv=-57;cid=testClusterID;nsid=1416426980;c=1731857797236), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-17T15:36:39,095 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6ce30f36-a42e-0287-cfce-b614a9c6228b 2024-11-17T15:36:39,173 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6ce30f36-a42e-0287-cfce-b614a9c6228b/cluster_3bb18fcf-0ee3-9799-c677-a8d421ff97cf/zookeeper_0, clientPort=59618, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6ce30f36-a42e-0287-cfce-b614a9c6228b/cluster_3bb18fcf-0ee3-9799-c677-a8d421ff97cf/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6ce30f36-a42e-0287-cfce-b614a9c6228b/cluster_3bb18fcf-0ee3-9799-c677-a8d421ff97cf/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-17T15:36:39,184 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=59618 2024-11-17T15:36:39,198 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-17T15:36:39,200 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-17T15:36:39,420 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073741825_1001 (size=7) 2024-11-17T15:36:39,827 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206 with version=8 2024-11-17T15:36:39,827 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/hbase-staging 2024-11-17T15:36:39,926 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-17T15:36:40,175 INFO [Time-limited test {}] client.ConnectionUtils(128): master/7a780d55532c:0 server-side Connection retries=45 2024-11-17T15:36:40,186 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-17T15:36:40,186 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-17T15:36:40,190 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-17T15:36:40,191 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-17T15:36:40,191 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-17T15:36:40,326 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-17T15:36:40,386 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-17T15:36:40,394 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-17T15:36:40,398 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-17T15:36:40,425 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 24859 (auto-detected) 2024-11-17T15:36:40,426 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-17T15:36:40,444 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:44223 2024-11-17T15:36:40,466 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:44223 connecting to ZooKeeper ensemble=127.0.0.1:59618 2024-11-17T15:36:40,498 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:442230x0, quorum=127.0.0.1:59618, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-17T15:36:40,500 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:44223-0x1012692c2820000 connected 2024-11-17T15:36:40,530 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-17T15:36:40,533 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-17T15:36:40,543 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:44223-0x1012692c2820000, quorum=127.0.0.1:59618, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-17T15:36:40,547 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206, hbase.cluster.distributed=false 2024-11-17T15:36:40,570 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:44223-0x1012692c2820000, quorum=127.0.0.1:59618, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-17T15:36:40,575 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=44223 2024-11-17T15:36:40,576 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=44223 2024-11-17T15:36:40,576 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=44223 2024-11-17T15:36:40,577 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=44223 2024-11-17T15:36:40,577 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=44223 2024-11-17T15:36:40,687 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/7a780d55532c:0 server-side Connection retries=45 2024-11-17T15:36:40,689 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-17T15:36:40,689 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-17T15:36:40,689 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-17T15:36:40,690 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-17T15:36:40,690 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-17T15:36:40,693 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-17T15:36:40,695 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-17T15:36:40,696 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:32955 2024-11-17T15:36:40,697 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:32955 connecting to ZooKeeper ensemble=127.0.0.1:59618 2024-11-17T15:36:40,699 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-17T15:36:40,701 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-17T15:36:40,709 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:329550x0, quorum=127.0.0.1:59618, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-17T15:36:40,710 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:329550x0, quorum=127.0.0.1:59618, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-17T15:36:40,710 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:32955-0x1012692c2820001 connected 2024-11-17T15:36:40,714 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-17T15:36:40,722 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-17T15:36:40,724 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:32955-0x1012692c2820001, quorum=127.0.0.1:59618, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-17T15:36:40,730 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:32955-0x1012692c2820001, quorum=127.0.0.1:59618, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-17T15:36:40,730 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=32955 2024-11-17T15:36:40,731 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=32955 2024-11-17T15:36:40,732 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=32955 2024-11-17T15:36:40,737 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=32955 2024-11-17T15:36:40,737 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=32955 2024-11-17T15:36:40,752 DEBUG [M:0;7a780d55532c:44223 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;7a780d55532c:44223 2024-11-17T15:36:40,753 INFO [master/7a780d55532c:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/7a780d55532c,44223,1731857799978 2024-11-17T15:36:40,761 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32955-0x1012692c2820001, quorum=127.0.0.1:59618, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-17T15:36:40,761 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44223-0x1012692c2820000, quorum=127.0.0.1:59618, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-17T15:36:40,763 DEBUG [master/7a780d55532c:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:44223-0x1012692c2820000, quorum=127.0.0.1:59618, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/7a780d55532c,44223,1731857799978 2024-11-17T15:36:40,787 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44223-0x1012692c2820000, quorum=127.0.0.1:59618, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T15:36:40,787 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32955-0x1012692c2820001, quorum=127.0.0.1:59618, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-17T15:36:40,788 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32955-0x1012692c2820001, quorum=127.0.0.1:59618, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T15:36:40,788 DEBUG [master/7a780d55532c:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:44223-0x1012692c2820000, quorum=127.0.0.1:59618, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-17T15:36:40,789 INFO [master/7a780d55532c:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/7a780d55532c,44223,1731857799978 from backup master directory 2024-11-17T15:36:40,792 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44223-0x1012692c2820000, quorum=127.0.0.1:59618, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/7a780d55532c,44223,1731857799978 2024-11-17T15:36:40,792 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32955-0x1012692c2820001, quorum=127.0.0.1:59618, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-17T15:36:40,792 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44223-0x1012692c2820000, quorum=127.0.0.1:59618, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-17T15:36:40,793 WARN [master/7a780d55532c:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-17T15:36:40,793 INFO [master/7a780d55532c:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=7a780d55532c,44223,1731857799978 2024-11-17T15:36:40,796 INFO [master/7a780d55532c:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-17T15:36:40,797 INFO [master/7a780d55532c:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-17T15:36:40,855 DEBUG [master/7a780d55532c:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/hbase.id] with ID: e01f4cd2-8971-4a73-861a-2ca0deb4f76e 2024-11-17T15:36:40,856 DEBUG [master/7a780d55532c:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/.tmp/hbase.id 2024-11-17T15:36:40,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073741826_1002 (size=42) 2024-11-17T15:36:41,269 DEBUG [master/7a780d55532c:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/.tmp/hbase.id]:[hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/hbase.id] 2024-11-17T15:36:41,321 INFO [master/7a780d55532c:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-17T15:36:41,327 INFO [master/7a780d55532c:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-17T15:36:41,354 INFO [master/7a780d55532c:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 24ms. 2024-11-17T15:36:41,358 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32955-0x1012692c2820001, quorum=127.0.0.1:59618, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T15:36:41,358 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44223-0x1012692c2820000, quorum=127.0.0.1:59618, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T15:36:41,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073741827_1003 (size=196) 2024-11-17T15:36:41,795 INFO [master/7a780d55532c:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-17T15:36:41,798 INFO [master/7a780d55532c:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-17T15:36:41,819 DEBUG [master/7a780d55532c:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:36:41,823 INFO [master/7a780d55532c:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-17T15:36:41,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073741828_1004 (size=1189) 2024-11-17T15:36:42,282 INFO [master/7a780d55532c:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/MasterData/data/master/store 2024-11-17T15:36:42,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073741829_1005 (size=34) 2024-11-17T15:36:42,310 INFO [master/7a780d55532c:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-17T15:36:42,314 DEBUG [master/7a780d55532c:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T15:36:42,316 DEBUG [master/7a780d55532c:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-17T15:36:42,316 INFO [master/7a780d55532c:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-17T15:36:42,316 DEBUG [master/7a780d55532c:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-17T15:36:42,318 DEBUG [master/7a780d55532c:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-17T15:36:42,318 DEBUG [master/7a780d55532c:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-17T15:36:42,318 INFO [master/7a780d55532c:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-17T15:36:42,320 DEBUG [master/7a780d55532c:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731857802315Disabling compacts and flushes for region at 1731857802315Disabling writes for close at 1731857802318 (+3 ms)Writing region close event to WAL at 1731857802318Closed at 1731857802318 2024-11-17T15:36:42,323 WARN [master/7a780d55532c:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/MasterData/data/master/store/.initializing 2024-11-17T15:36:42,323 DEBUG [master/7a780d55532c:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/MasterData/WALs/7a780d55532c,44223,1731857799978 2024-11-17T15:36:42,334 INFO [master/7a780d55532c:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-17T15:36:42,352 INFO [master/7a780d55532c:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=7a780d55532c%2C44223%2C1731857799978, suffix=, logDir=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/MasterData/WALs/7a780d55532c,44223,1731857799978, archiveDir=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/MasterData/oldWALs, maxLogs=10 2024-11-17T15:36:42,376 DEBUG [master/7a780d55532c:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/MasterData/WALs/7a780d55532c,44223,1731857799978/7a780d55532c%2C44223%2C1731857799978.1731857802357, exclude list is [], retry=0 2024-11-17T15:36:42,394 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46871,DS-f414f29e-7b04-4193-b099-6d2b4232d69f,DISK] 2024-11-17T15:36:42,398 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-17T15:36:42,442 INFO [master/7a780d55532c:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/MasterData/WALs/7a780d55532c,44223,1731857799978/7a780d55532c%2C44223%2C1731857799978.1731857802357 2024-11-17T15:36:42,443 DEBUG [master/7a780d55532c:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39733:39733)] 2024-11-17T15:36:42,444 DEBUG [master/7a780d55532c:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-17T15:36:42,444 DEBUG [master/7a780d55532c:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T15:36:42,448 DEBUG [master/7a780d55532c:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-17T15:36:42,450 DEBUG [master/7a780d55532c:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-17T15:36:42,491 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-17T15:36:42,518 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-17T15:36:42,523 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:36:42,526 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-17T15:36:42,526 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-17T15:36:42,530 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-17T15:36:42,531 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:36:42,532 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T15:36:42,532 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-17T15:36:42,536 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-17T15:36:42,536 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:36:42,537 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T15:36:42,538 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-17T15:36:42,542 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-17T15:36:42,542 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:36:42,543 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T15:36:42,543 DEBUG [master/7a780d55532c:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-17T15:36:42,548 DEBUG [master/7a780d55532c:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-17T15:36:42,549 DEBUG [master/7a780d55532c:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-17T15:36:42,555 DEBUG [master/7a780d55532c:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-17T15:36:42,556 DEBUG [master/7a780d55532c:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-17T15:36:42,560 DEBUG [master/7a780d55532c:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-17T15:36:42,565 DEBUG [master/7a780d55532c:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-17T15:36:42,570 DEBUG [master/7a780d55532c:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-17T15:36:42,571 INFO [master/7a780d55532c:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65779722, jitterRate=-0.01980575919151306}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-17T15:36:42,577 DEBUG [master/7a780d55532c:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731857802465Initializing all the Stores at 1731857802468 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731857802468Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731857802469 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731857802469Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731857802469Cleaning up temporary data from old regions at 1731857802556 (+87 ms)Region opened successfully at 1731857802577 (+21 ms) 2024-11-17T15:36:42,579 INFO [master/7a780d55532c:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-17T15:36:42,619 DEBUG [master/7a780d55532c:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1107980a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=7a780d55532c/172.17.0.2:0 2024-11-17T15:36:42,660 INFO [master/7a780d55532c:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-17T15:36:42,671 INFO [master/7a780d55532c:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-17T15:36:42,672 INFO [master/7a780d55532c:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-17T15:36:42,675 INFO [master/7a780d55532c:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-17T15:36:42,676 INFO [master/7a780d55532c:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-17T15:36:42,682 INFO [master/7a780d55532c:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-17T15:36:42,682 INFO [master/7a780d55532c:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-17T15:36:42,714 INFO [master/7a780d55532c:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-17T15:36:42,724 DEBUG [master/7a780d55532c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44223-0x1012692c2820000, quorum=127.0.0.1:59618, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-17T15:36:42,726 DEBUG [master/7a780d55532c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-17T15:36:42,730 INFO [master/7a780d55532c:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-17T15:36:42,732 DEBUG [master/7a780d55532c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44223-0x1012692c2820000, quorum=127.0.0.1:59618, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-17T15:36:42,736 DEBUG [master/7a780d55532c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-17T15:36:42,739 INFO [master/7a780d55532c:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-17T15:36:42,743 DEBUG [master/7a780d55532c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44223-0x1012692c2820000, quorum=127.0.0.1:59618, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-17T15:36:42,744 DEBUG [master/7a780d55532c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-17T15:36:42,746 DEBUG [master/7a780d55532c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44223-0x1012692c2820000, quorum=127.0.0.1:59618, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-17T15:36:42,747 DEBUG [master/7a780d55532c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-17T15:36:42,765 DEBUG [master/7a780d55532c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44223-0x1012692c2820000, quorum=127.0.0.1:59618, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-17T15:36:42,766 DEBUG [master/7a780d55532c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-17T15:36:42,772 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44223-0x1012692c2820000, quorum=127.0.0.1:59618, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-17T15:36:42,772 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32955-0x1012692c2820001, quorum=127.0.0.1:59618, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-17T15:36:42,772 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44223-0x1012692c2820000, quorum=127.0.0.1:59618, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T15:36:42,773 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32955-0x1012692c2820001, quorum=127.0.0.1:59618, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T15:36:42,776 INFO [master/7a780d55532c:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=7a780d55532c,44223,1731857799978, sessionid=0x1012692c2820000, setting cluster-up flag (Was=false) 2024-11-17T15:36:42,791 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44223-0x1012692c2820000, quorum=127.0.0.1:59618, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T15:36:42,791 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32955-0x1012692c2820001, quorum=127.0.0.1:59618, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T15:36:42,799 DEBUG [master/7a780d55532c:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-17T15:36:42,801 DEBUG [master/7a780d55532c:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=7a780d55532c,44223,1731857799978 2024-11-17T15:36:42,807 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44223-0x1012692c2820000, quorum=127.0.0.1:59618, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T15:36:42,807 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32955-0x1012692c2820001, quorum=127.0.0.1:59618, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T15:36:42,814 DEBUG [master/7a780d55532c:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-17T15:36:42,816 DEBUG [master/7a780d55532c:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=7a780d55532c,44223,1731857799978 2024-11-17T15:36:42,823 INFO [master/7a780d55532c:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-17T15:36:42,843 INFO [RS:0;7a780d55532c:32955 {}] regionserver.HRegionServer(746): ClusterId : e01f4cd2-8971-4a73-861a-2ca0deb4f76e 2024-11-17T15:36:42,847 DEBUG [RS:0;7a780d55532c:32955 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-17T15:36:42,853 DEBUG [RS:0;7a780d55532c:32955 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-17T15:36:42,853 DEBUG [RS:0;7a780d55532c:32955 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-17T15:36:42,858 DEBUG [RS:0;7a780d55532c:32955 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-17T15:36:42,858 DEBUG [RS:0;7a780d55532c:32955 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4cdd726e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=7a780d55532c/172.17.0.2:0 2024-11-17T15:36:42,879 DEBUG [RS:0;7a780d55532c:32955 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;7a780d55532c:32955 2024-11-17T15:36:42,882 INFO [RS:0;7a780d55532c:32955 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-17T15:36:42,883 INFO [RS:0;7a780d55532c:32955 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-17T15:36:42,883 DEBUG [RS:0;7a780d55532c:32955 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-17T15:36:42,886 INFO [RS:0;7a780d55532c:32955 {}] regionserver.HRegionServer(2659): reportForDuty to master=7a780d55532c,44223,1731857799978 with port=32955, startcode=1731857800648 2024-11-17T15:36:42,898 DEBUG [RS:0;7a780d55532c:32955 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-17T15:36:42,918 DEBUG [master/7a780d55532c:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-17T15:36:42,931 INFO [master/7a780d55532c:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-17T15:36:42,942 INFO [master/7a780d55532c:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-17T15:36:42,942 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34373, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-17T15:36:42,949 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44223 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:36:42,953 DEBUG [master/7a780d55532c:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 7a780d55532c,44223,1731857799978 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-17T15:36:42,961 DEBUG [master/7a780d55532c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/7a780d55532c:0, corePoolSize=5, maxPoolSize=5 2024-11-17T15:36:42,962 DEBUG [master/7a780d55532c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/7a780d55532c:0, corePoolSize=5, maxPoolSize=5 2024-11-17T15:36:42,962 DEBUG [master/7a780d55532c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/7a780d55532c:0, corePoolSize=5, maxPoolSize=5 2024-11-17T15:36:42,962 DEBUG [master/7a780d55532c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/7a780d55532c:0, corePoolSize=5, maxPoolSize=5 2024-11-17T15:36:42,962 DEBUG [master/7a780d55532c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/7a780d55532c:0, corePoolSize=10, maxPoolSize=10 2024-11-17T15:36:42,963 DEBUG [master/7a780d55532c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/7a780d55532c:0, corePoolSize=1, maxPoolSize=1 2024-11-17T15:36:42,963 DEBUG [master/7a780d55532c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/7a780d55532c:0, corePoolSize=2, maxPoolSize=2 2024-11-17T15:36:42,963 DEBUG [master/7a780d55532c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/7a780d55532c:0, corePoolSize=1, maxPoolSize=1 2024-11-17T15:36:42,974 INFO [master/7a780d55532c:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731857832974 2024-11-17T15:36:42,975 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-17T15:36:42,976 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-17T15:36:42,976 INFO [master/7a780d55532c:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-17T15:36:42,977 INFO [master/7a780d55532c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-17T15:36:42,982 INFO [master/7a780d55532c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-17T15:36:42,982 INFO [master/7a780d55532c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-17T15:36:42,983 INFO [master/7a780d55532c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-17T15:36:42,984 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:36:42,983 INFO [master/7a780d55532c:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-17T15:36:42,984 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-17T15:36:42,988 INFO [master/7a780d55532c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-17T15:36:42,993 DEBUG [RS:0;7a780d55532c:32955 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-17T15:36:42,993 WARN [RS:0;7a780d55532c:32955 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-17T15:36:42,994 INFO [master/7a780d55532c:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-17T15:36:42,995 INFO [master/7a780d55532c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-17T15:36:42,996 INFO [master/7a780d55532c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-17T15:36:43,001 INFO [master/7a780d55532c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-17T15:36:43,001 INFO [master/7a780d55532c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-17T15:36:43,004 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073741831_1007 (size=1321) 2024-11-17T15:36:43,008 DEBUG [master/7a780d55532c:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/7a780d55532c:0:becomeActiveMaster-HFileCleaner.large.0-1731857803002,5,FailOnTimeoutGroup] 2024-11-17T15:36:43,012 DEBUG [master/7a780d55532c:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/7a780d55532c:0:becomeActiveMaster-HFileCleaner.small.0-1731857803009,5,FailOnTimeoutGroup] 2024-11-17T15:36:43,012 INFO [master/7a780d55532c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-17T15:36:43,013 INFO [master/7a780d55532c:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-17T15:36:43,014 INFO [master/7a780d55532c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-17T15:36:43,014 INFO [master/7a780d55532c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-17T15:36:43,095 INFO [RS:0;7a780d55532c:32955 {}] regionserver.HRegionServer(2659): reportForDuty to master=7a780d55532c,44223,1731857799978 with port=32955, startcode=1731857800648 2024-11-17T15:36:43,098 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44223 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 7a780d55532c,32955,1731857800648 2024-11-17T15:36:43,101 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44223 {}] master.ServerManager(517): Registering regionserver=7a780d55532c,32955,1731857800648 2024-11-17T15:36:43,111 DEBUG [RS:0;7a780d55532c:32955 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206 2024-11-17T15:36:43,112 DEBUG [RS:0;7a780d55532c:32955 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:39013 2024-11-17T15:36:43,112 DEBUG [RS:0;7a780d55532c:32955 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-17T15:36:43,116 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44223-0x1012692c2820000, quorum=127.0.0.1:59618, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-17T15:36:43,117 DEBUG [RS:0;7a780d55532c:32955 {}] zookeeper.ZKUtil(111): regionserver:32955-0x1012692c2820001, quorum=127.0.0.1:59618, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/7a780d55532c,32955,1731857800648 2024-11-17T15:36:43,117 WARN [RS:0;7a780d55532c:32955 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-17T15:36:43,118 INFO [RS:0;7a780d55532c:32955 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-17T15:36:43,118 DEBUG [RS:0;7a780d55532c:32955 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/WALs/7a780d55532c,32955,1731857800648 2024-11-17T15:36:43,122 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [7a780d55532c,32955,1731857800648] 2024-11-17T15:36:43,147 INFO [RS:0;7a780d55532c:32955 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-17T15:36:43,162 INFO [RS:0;7a780d55532c:32955 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-17T15:36:43,167 INFO [RS:0;7a780d55532c:32955 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-17T15:36:43,167 INFO [RS:0;7a780d55532c:32955 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-17T15:36:43,168 INFO [RS:0;7a780d55532c:32955 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-17T15:36:43,173 INFO [RS:0;7a780d55532c:32955 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-17T15:36:43,175 INFO [RS:0;7a780d55532c:32955 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-17T15:36:43,175 DEBUG [RS:0;7a780d55532c:32955 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/7a780d55532c:0, corePoolSize=1, maxPoolSize=1 2024-11-17T15:36:43,176 DEBUG [RS:0;7a780d55532c:32955 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/7a780d55532c:0, corePoolSize=1, maxPoolSize=1 2024-11-17T15:36:43,176 DEBUG [RS:0;7a780d55532c:32955 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/7a780d55532c:0, corePoolSize=1, maxPoolSize=1 2024-11-17T15:36:43,176 DEBUG [RS:0;7a780d55532c:32955 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/7a780d55532c:0, corePoolSize=1, maxPoolSize=1 2024-11-17T15:36:43,176 DEBUG [RS:0;7a780d55532c:32955 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/7a780d55532c:0, corePoolSize=1, maxPoolSize=1 2024-11-17T15:36:43,176 DEBUG [RS:0;7a780d55532c:32955 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/7a780d55532c:0, corePoolSize=2, maxPoolSize=2 2024-11-17T15:36:43,177 DEBUG [RS:0;7a780d55532c:32955 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/7a780d55532c:0, corePoolSize=1, maxPoolSize=1 2024-11-17T15:36:43,177 DEBUG [RS:0;7a780d55532c:32955 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/7a780d55532c:0, corePoolSize=1, maxPoolSize=1 2024-11-17T15:36:43,177 DEBUG [RS:0;7a780d55532c:32955 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/7a780d55532c:0, corePoolSize=1, maxPoolSize=1 2024-11-17T15:36:43,177 DEBUG [RS:0;7a780d55532c:32955 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/7a780d55532c:0, corePoolSize=1, maxPoolSize=1 2024-11-17T15:36:43,177 DEBUG [RS:0;7a780d55532c:32955 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/7a780d55532c:0, corePoolSize=1, maxPoolSize=1 2024-11-17T15:36:43,178 DEBUG [RS:0;7a780d55532c:32955 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/7a780d55532c:0, corePoolSize=1, maxPoolSize=1 2024-11-17T15:36:43,178 DEBUG [RS:0;7a780d55532c:32955 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/7a780d55532c:0, corePoolSize=3, maxPoolSize=3 2024-11-17T15:36:43,178 DEBUG [RS:0;7a780d55532c:32955 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0, corePoolSize=3, maxPoolSize=3 2024-11-17T15:36:43,179 INFO [RS:0;7a780d55532c:32955 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-17T15:36:43,179 INFO [RS:0;7a780d55532c:32955 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-17T15:36:43,179 INFO [RS:0;7a780d55532c:32955 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-17T15:36:43,179 INFO [RS:0;7a780d55532c:32955 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-17T15:36:43,179 INFO [RS:0;7a780d55532c:32955 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-17T15:36:43,180 INFO [RS:0;7a780d55532c:32955 {}] hbase.ChoreService(168): Chore ScheduledChore name=7a780d55532c,32955,1731857800648-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-17T15:36:43,200 INFO [RS:0;7a780d55532c:32955 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-17T15:36:43,202 INFO [RS:0;7a780d55532c:32955 {}] hbase.ChoreService(168): Chore ScheduledChore name=7a780d55532c,32955,1731857800648-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-17T15:36:43,203 INFO [RS:0;7a780d55532c:32955 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-17T15:36:43,203 INFO [RS:0;7a780d55532c:32955 {}] regionserver.Replication(171): 7a780d55532c,32955,1731857800648 started 2024-11-17T15:36:43,223 INFO [RS:0;7a780d55532c:32955 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-17T15:36:43,223 INFO [RS:0;7a780d55532c:32955 {}] regionserver.HRegionServer(1482): Serving as 7a780d55532c,32955,1731857800648, RpcServer on 7a780d55532c/172.17.0.2:32955, sessionid=0x1012692c2820001 2024-11-17T15:36:43,224 DEBUG [RS:0;7a780d55532c:32955 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-17T15:36:43,224 DEBUG [RS:0;7a780d55532c:32955 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 7a780d55532c,32955,1731857800648 2024-11-17T15:36:43,224 DEBUG [RS:0;7a780d55532c:32955 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '7a780d55532c,32955,1731857800648' 2024-11-17T15:36:43,224 DEBUG [RS:0;7a780d55532c:32955 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-17T15:36:43,225 DEBUG [RS:0;7a780d55532c:32955 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-17T15:36:43,226 DEBUG [RS:0;7a780d55532c:32955 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-17T15:36:43,226 DEBUG [RS:0;7a780d55532c:32955 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-17T15:36:43,226 DEBUG [RS:0;7a780d55532c:32955 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 7a780d55532c,32955,1731857800648 2024-11-17T15:36:43,226 DEBUG [RS:0;7a780d55532c:32955 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '7a780d55532c,32955,1731857800648' 2024-11-17T15:36:43,227 DEBUG [RS:0;7a780d55532c:32955 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-17T15:36:43,227 DEBUG [RS:0;7a780d55532c:32955 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-17T15:36:43,228 DEBUG [RS:0;7a780d55532c:32955 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-17T15:36:43,228 INFO [RS:0;7a780d55532c:32955 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-17T15:36:43,228 INFO [RS:0;7a780d55532c:32955 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-17T15:36:43,336 INFO [RS:0;7a780d55532c:32955 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-17T15:36:43,340 INFO [RS:0;7a780d55532c:32955 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=7a780d55532c%2C32955%2C1731857800648, suffix=, logDir=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/WALs/7a780d55532c,32955,1731857800648, archiveDir=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/oldWALs, maxLogs=32 2024-11-17T15:36:43,358 DEBUG [RS:0;7a780d55532c:32955 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/WALs/7a780d55532c,32955,1731857800648/7a780d55532c%2C32955%2C1731857800648.1731857803343, exclude list is [], retry=0 2024-11-17T15:36:43,363 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46871,DS-f414f29e-7b04-4193-b099-6d2b4232d69f,DISK] 2024-11-17T15:36:43,368 INFO [RS:0;7a780d55532c:32955 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/WALs/7a780d55532c,32955,1731857800648/7a780d55532c%2C32955%2C1731857800648.1731857803343 2024-11-17T15:36:43,368 DEBUG [RS:0;7a780d55532c:32955 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39733:39733)] 2024-11-17T15:36:43,407 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-17T15:36:43,408 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206 2024-11-17T15:36:43,423 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073741833_1009 (size=32) 2024-11-17T15:36:43,826 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T15:36:43,829 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-17T15:36:43,832 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-17T15:36:43,832 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:36:43,833 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-17T15:36:43,834 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-17T15:36:43,836 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-17T15:36:43,836 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:36:43,837 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-17T15:36:43,838 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-17T15:36:43,840 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-17T15:36:43,840 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:36:43,841 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-17T15:36:43,841 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-17T15:36:43,844 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-17T15:36:43,844 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:36:43,845 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-17T15:36:43,845 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-17T15:36:43,847 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/hbase/meta/1588230740 2024-11-17T15:36:43,848 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/hbase/meta/1588230740 2024-11-17T15:36:43,851 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-17T15:36:43,851 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-17T15:36:43,852 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-17T15:36:43,855 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-17T15:36:43,859 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-17T15:36:43,861 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65964122, jitterRate=-0.017057985067367554}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-17T15:36:43,864 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731857803827Initializing all the Stores at 1731857803829 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731857803829Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731857803829Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731857803829Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731857803829Cleaning up temporary data from old regions at 1731857803851 (+22 ms)Region opened successfully at 1731857803864 (+13 ms) 2024-11-17T15:36:43,864 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-17T15:36:43,864 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-17T15:36:43,864 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-17T15:36:43,864 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-17T15:36:43,865 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-17T15:36:43,866 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-17T15:36:43,866 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731857803864Disabling compacts and flushes for region at 1731857803864Disabling writes for close at 1731857803864Writing region close event to WAL at 1731857803866 (+2 ms)Closed at 1731857803866 2024-11-17T15:36:43,870 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-17T15:36:43,871 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-17T15:36:43,877 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-17T15:36:43,886 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-17T15:36:43,889 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-17T15:36:44,042 DEBUG [7a780d55532c:44223 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-17T15:36:44,055 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=7a780d55532c,32955,1731857800648 2024-11-17T15:36:44,063 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 7a780d55532c,32955,1731857800648, state=OPENING 2024-11-17T15:36:44,068 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-17T15:36:44,070 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44223-0x1012692c2820000, quorum=127.0.0.1:59618, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T15:36:44,070 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32955-0x1012692c2820001, quorum=127.0.0.1:59618, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T15:36:44,072 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-17T15:36:44,072 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-17T15:36:44,073 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-17T15:36:44,076 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=7a780d55532c,32955,1731857800648}] 2024-11-17T15:36:44,258 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-17T15:36:44,262 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53475, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-17T15:36:44,272 INFO [RS_OPEN_META-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-17T15:36:44,273 INFO [RS_OPEN_META-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-17T15:36:44,274 INFO [RS_OPEN_META-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-17T15:36:44,280 INFO [RS_OPEN_META-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=7a780d55532c%2C32955%2C1731857800648.meta, suffix=.meta, logDir=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/WALs/7a780d55532c,32955,1731857800648, archiveDir=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/oldWALs, maxLogs=32 2024-11-17T15:36:44,305 DEBUG [RS_OPEN_META-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/WALs/7a780d55532c,32955,1731857800648/7a780d55532c%2C32955%2C1731857800648.meta.1731857804283.meta, exclude list is [], retry=0 2024-11-17T15:36:44,309 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46871,DS-f414f29e-7b04-4193-b099-6d2b4232d69f,DISK] 2024-11-17T15:36:44,313 INFO [RS_OPEN_META-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/WALs/7a780d55532c,32955,1731857800648/7a780d55532c%2C32955%2C1731857800648.meta.1731857804283.meta 2024-11-17T15:36:44,320 DEBUG [RS_OPEN_META-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39733:39733)] 2024-11-17T15:36:44,321 DEBUG [RS_OPEN_META-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-17T15:36:44,323 DEBUG [RS_OPEN_META-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-17T15:36:44,327 DEBUG [RS_OPEN_META-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-17T15:36:44,334 INFO [RS_OPEN_META-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-17T15:36:44,340 DEBUG [RS_OPEN_META-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-17T15:36:44,341 DEBUG [RS_OPEN_META-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T15:36:44,341 DEBUG [RS_OPEN_META-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-17T15:36:44,341 DEBUG [RS_OPEN_META-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-17T15:36:44,346 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-17T15:36:44,348 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-17T15:36:44,348 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:36:44,349 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-17T15:36:44,349 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-17T15:36:44,351 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-17T15:36:44,352 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:36:44,353 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-17T15:36:44,353 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-17T15:36:44,355 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-17T15:36:44,355 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:36:44,356 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-17T15:36:44,356 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-17T15:36:44,358 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-17T15:36:44,358 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:36:44,359 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-17T15:36:44,360 DEBUG [RS_OPEN_META-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-17T15:36:44,361 DEBUG [RS_OPEN_META-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/hbase/meta/1588230740 2024-11-17T15:36:44,364 DEBUG [RS_OPEN_META-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/hbase/meta/1588230740 2024-11-17T15:36:44,367 DEBUG [RS_OPEN_META-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-17T15:36:44,367 DEBUG [RS_OPEN_META-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-17T15:36:44,369 DEBUG [RS_OPEN_META-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-17T15:36:44,372 DEBUG [RS_OPEN_META-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-17T15:36:44,374 INFO [RS_OPEN_META-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61385191, jitterRate=-0.0852893739938736}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-17T15:36:44,374 DEBUG [RS_OPEN_META-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-17T15:36:44,375 DEBUG [RS_OPEN_META-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731857804342Writing region info on filesystem at 1731857804342Initializing all the Stores at 1731857804345 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731857804345Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731857804345Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731857804345Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731857804345Cleaning up temporary data from old regions at 1731857804367 (+22 ms)Running coprocessor post-open hooks at 1731857804374 (+7 ms)Region opened successfully at 1731857804375 (+1 ms) 2024-11-17T15:36:44,384 INFO [RS_OPEN_META-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731857804246 2024-11-17T15:36:44,396 DEBUG [RS_OPEN_META-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-17T15:36:44,397 INFO [RS_OPEN_META-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-17T15:36:44,399 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=7a780d55532c,32955,1731857800648 2024-11-17T15:36:44,401 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 7a780d55532c,32955,1731857800648, state=OPEN 2024-11-17T15:36:44,409 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44223-0x1012692c2820000, quorum=127.0.0.1:59618, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-17T15:36:44,409 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32955-0x1012692c2820001, quorum=127.0.0.1:59618, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-17T15:36:44,409 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-17T15:36:44,410 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-17T15:36:44,410 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=7a780d55532c,32955,1731857800648 2024-11-17T15:36:44,416 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-17T15:36:44,417 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=7a780d55532c,32955,1731857800648 in 335 msec 2024-11-17T15:36:44,430 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-17T15:36:44,430 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 542 msec 2024-11-17T15:36:44,432 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-17T15:36:44,432 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-17T15:36:44,455 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T15:36:44,457 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7a780d55532c,32955,1731857800648, seqNum=-1] 2024-11-17T15:36:44,484 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T15:36:44,487 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58795, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T15:36:44,507 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.6520 sec 2024-11-17T15:36:44,507 INFO [master/7a780d55532c:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731857804507, completionTime=-1 2024-11-17T15:36:44,511 INFO [master/7a780d55532c:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-17T15:36:44,511 DEBUG [master/7a780d55532c:0:becomeActiveMaster {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-17T15:36:44,576 INFO [master/7a780d55532c:0:becomeActiveMaster {}] assignment.AssignmentManager(1768): Number of RegionServers=1 2024-11-17T15:36:44,576 INFO [master/7a780d55532c:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731857864576 2024-11-17T15:36:44,576 INFO [master/7a780d55532c:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731857924576 2024-11-17T15:36:44,576 INFO [master/7a780d55532c:0:becomeActiveMaster {}] assignment.AssignmentManager(1775): Joined the cluster in 64 msec 2024-11-17T15:36:44,578 INFO [master/7a780d55532c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7a780d55532c,44223,1731857799978-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-17T15:36:44,578 INFO [master/7a780d55532c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7a780d55532c,44223,1731857799978-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-17T15:36:44,579 INFO [master/7a780d55532c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7a780d55532c,44223,1731857799978-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-17T15:36:44,580 INFO [master/7a780d55532c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-7a780d55532c:44223, period=300000, unit=MILLISECONDS is enabled. 2024-11-17T15:36:44,581 INFO [master/7a780d55532c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-17T15:36:44,581 INFO [master/7a780d55532c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-17T15:36:44,588 DEBUG [master/7a780d55532c:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-17T15:36:44,611 INFO [master/7a780d55532c:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 3.818sec 2024-11-17T15:36:44,612 INFO [master/7a780d55532c:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-17T15:36:44,613 INFO [master/7a780d55532c:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-17T15:36:44,614 INFO [master/7a780d55532c:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-17T15:36:44,615 INFO [master/7a780d55532c:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-17T15:36:44,615 INFO [master/7a780d55532c:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-17T15:36:44,616 INFO [master/7a780d55532c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7a780d55532c,44223,1731857799978-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-17T15:36:44,616 INFO [master/7a780d55532c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7a780d55532c,44223,1731857799978-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-17T15:36:44,624 DEBUG [master/7a780d55532c:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-17T15:36:44,625 INFO [master/7a780d55532c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-17T15:36:44,626 INFO [master/7a780d55532c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7a780d55532c,44223,1731857799978-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-17T15:36:44,659 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@343aff6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:36:44,661 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-17T15:36:44,662 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-17T15:36:44,665 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7a780d55532c,44223,-1 for getting cluster id 2024-11-17T15:36:44,667 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T15:36:44,675 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e01f4cd2-8971-4a73-861a-2ca0deb4f76e' 2024-11-17T15:36:44,677 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T15:36:44,678 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e01f4cd2-8971-4a73-861a-2ca0deb4f76e" 2024-11-17T15:36:44,680 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a1c4f13, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:36:44,680 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7a780d55532c,44223,-1] 2024-11-17T15:36:44,682 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T15:36:44,684 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:36:44,686 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41638, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T15:36:44,689 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32372268, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:36:44,689 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T15:36:44,696 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7a780d55532c,32955,1731857800648, seqNum=-1] 2024-11-17T15:36:44,696 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T15:36:44,699 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57462, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T15:36:44,721 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=7a780d55532c,44223,1731857799978 2024-11-17T15:36:44,737 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMixedAtomicity Thread=211, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=223, ProcessCount=11, AvailableMemoryMB=3925 2024-11-17T15:36:44,747 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-17T15:36:44,752 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 7a780d55532c,44223,1731857799978 2024-11-17T15:36:44,755 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@48fe66d7 2024-11-17T15:36:44,756 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-17T15:36:44,758 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41652, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-17T15:36:44,760 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-17T15:36:44,764 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-17T15:36:44,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-17T15:36:44,775 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-17T15:36:44,777 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-11-17T15:36:44,777 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:36:44,780 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-17T15:36:44,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-17T15:36:44,791 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073741835_1011 (size=959) 2024-11-17T15:36:44,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-17T15:36:45,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-17T15:36:45,195 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => a80c4e623aab44bea3cfa33ee4fee8f3, NAME => 'TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206 2024-11-17T15:36:45,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073741836_1012 (size=53) 2024-11-17T15:36:45,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-17T15:36:45,606 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T15:36:45,606 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing a80c4e623aab44bea3cfa33ee4fee8f3, disabling compactions & flushes 2024-11-17T15:36:45,606 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3. 2024-11-17T15:36:45,606 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3. 2024-11-17T15:36:45,606 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3. after waiting 0 ms 2024-11-17T15:36:45,606 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3. 2024-11-17T15:36:45,606 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3. 2024-11-17T15:36:45,606 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for a80c4e623aab44bea3cfa33ee4fee8f3: Waiting for close lock at 1731857805606Disabling compacts and flushes for region at 1731857805606Disabling writes for close at 1731857805606Writing region close event to WAL at 1731857805606Closed at 1731857805606 2024-11-17T15:36:45,609 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-17T15:36:45,616 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731857805609"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731857805609"}]},"ts":"1731857805609"} 2024-11-17T15:36:45,622 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-17T15:36:45,624 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-17T15:36:45,628 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731857805624"}]},"ts":"1731857805624"} 2024-11-17T15:36:45,633 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-17T15:36:45,635 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a80c4e623aab44bea3cfa33ee4fee8f3, ASSIGN}] 2024-11-17T15:36:45,638 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a80c4e623aab44bea3cfa33ee4fee8f3, ASSIGN 2024-11-17T15:36:45,640 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a80c4e623aab44bea3cfa33ee4fee8f3, ASSIGN; state=OFFLINE, location=7a780d55532c,32955,1731857800648; forceNewPlan=false, retain=false 2024-11-17T15:36:45,791 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=a80c4e623aab44bea3cfa33ee4fee8f3, regionState=OPENING, regionLocation=7a780d55532c,32955,1731857800648 2024-11-17T15:36:45,796 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a80c4e623aab44bea3cfa33ee4fee8f3, ASSIGN because future has completed 2024-11-17T15:36:45,797 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648}] 2024-11-17T15:36:45,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-17T15:36:45,958 INFO [RS_OPEN_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3. 2024-11-17T15:36:45,958 DEBUG [RS_OPEN_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => a80c4e623aab44bea3cfa33ee4fee8f3, NAME => 'TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.', STARTKEY => '', ENDKEY => ''} 2024-11-17T15:36:45,959 DEBUG [RS_OPEN_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees a80c4e623aab44bea3cfa33ee4fee8f3 2024-11-17T15:36:45,959 DEBUG [RS_OPEN_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T15:36:45,959 DEBUG [RS_OPEN_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for a80c4e623aab44bea3cfa33ee4fee8f3 2024-11-17T15:36:45,959 DEBUG [RS_OPEN_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for a80c4e623aab44bea3cfa33ee4fee8f3 2024-11-17T15:36:45,962 INFO [StoreOpener-a80c4e623aab44bea3cfa33ee4fee8f3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region a80c4e623aab44bea3cfa33ee4fee8f3 2024-11-17T15:36:45,965 INFO [StoreOpener-a80c4e623aab44bea3cfa33ee4fee8f3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a80c4e623aab44bea3cfa33ee4fee8f3 columnFamilyName A 2024-11-17T15:36:45,965 DEBUG [StoreOpener-a80c4e623aab44bea3cfa33ee4fee8f3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:36:45,966 INFO [StoreOpener-a80c4e623aab44bea3cfa33ee4fee8f3-1 {}] regionserver.HStore(327): Store=a80c4e623aab44bea3cfa33ee4fee8f3/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T15:36:45,966 INFO [StoreOpener-a80c4e623aab44bea3cfa33ee4fee8f3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region a80c4e623aab44bea3cfa33ee4fee8f3 2024-11-17T15:36:45,968 INFO [StoreOpener-a80c4e623aab44bea3cfa33ee4fee8f3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a80c4e623aab44bea3cfa33ee4fee8f3 columnFamilyName B 2024-11-17T15:36:45,968 DEBUG [StoreOpener-a80c4e623aab44bea3cfa33ee4fee8f3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:36:45,969 INFO [StoreOpener-a80c4e623aab44bea3cfa33ee4fee8f3-1 {}] regionserver.HStore(327): Store=a80c4e623aab44bea3cfa33ee4fee8f3/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T15:36:45,969 INFO [StoreOpener-a80c4e623aab44bea3cfa33ee4fee8f3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region a80c4e623aab44bea3cfa33ee4fee8f3 2024-11-17T15:36:45,971 INFO [StoreOpener-a80c4e623aab44bea3cfa33ee4fee8f3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a80c4e623aab44bea3cfa33ee4fee8f3 columnFamilyName C 2024-11-17T15:36:45,971 DEBUG [StoreOpener-a80c4e623aab44bea3cfa33ee4fee8f3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:36:45,972 INFO [StoreOpener-a80c4e623aab44bea3cfa33ee4fee8f3-1 {}] regionserver.HStore(327): Store=a80c4e623aab44bea3cfa33ee4fee8f3/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T15:36:45,972 DEBUG [RS_OPEN_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for a80c4e623aab44bea3cfa33ee4fee8f3 2024-11-17T15:36:45,973 DEBUG [RS_OPEN_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3 2024-11-17T15:36:45,974 DEBUG [RS_OPEN_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3 2024-11-17T15:36:45,976 DEBUG [RS_OPEN_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for a80c4e623aab44bea3cfa33ee4fee8f3 2024-11-17T15:36:45,976 DEBUG [RS_OPEN_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for a80c4e623aab44bea3cfa33ee4fee8f3 2024-11-17T15:36:45,977 DEBUG [RS_OPEN_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-17T15:36:45,979 DEBUG [RS_OPEN_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for a80c4e623aab44bea3cfa33ee4fee8f3 2024-11-17T15:36:45,983 DEBUG [RS_OPEN_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-17T15:36:45,984 INFO [RS_OPEN_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened a80c4e623aab44bea3cfa33ee4fee8f3; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74082269, jitterRate=0.10391183197498322}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-17T15:36:45,984 DEBUG [RS_OPEN_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a80c4e623aab44bea3cfa33ee4fee8f3 2024-11-17T15:36:45,985 DEBUG [RS_OPEN_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for a80c4e623aab44bea3cfa33ee4fee8f3: Running coprocessor pre-open hook at 1731857805960Writing region info on filesystem at 1731857805960Initializing all the Stores at 1731857805962 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731857805962Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731857805962Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731857805962Cleaning up temporary data from old regions at 1731857805976 (+14 ms)Running coprocessor post-open hooks at 1731857805984 (+8 ms)Region opened successfully at 1731857805985 (+1 ms) 2024-11-17T15:36:45,988 INFO [RS_OPEN_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., pid=6, masterSystemTime=1731857805951 2024-11-17T15:36:45,992 DEBUG [RS_OPEN_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3. 2024-11-17T15:36:45,992 INFO [RS_OPEN_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3. 2024-11-17T15:36:45,994 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=a80c4e623aab44bea3cfa33ee4fee8f3, regionState=OPEN, openSeqNum=2, regionLocation=7a780d55532c,32955,1731857800648 2024-11-17T15:36:46,004 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 because future has completed 2024-11-17T15:36:46,005 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44223 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=7a780d55532c,32955,1731857800648, table=TestAcidGuarantees, region=a80c4e623aab44bea3cfa33ee4fee8f3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-17T15:36:46,011 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-17T15:36:46,011 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 in 209 msec 2024-11-17T15:36:46,015 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-17T15:36:46,016 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a80c4e623aab44bea3cfa33ee4fee8f3, ASSIGN in 376 msec 2024-11-17T15:36:46,017 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-17T15:36:46,018 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731857806017"}]},"ts":"1731857806017"} 2024-11-17T15:36:46,022 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-17T15:36:46,024 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-17T15:36:46,028 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2570 sec 2024-11-17T15:36:46,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-17T15:36:46,935 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-17T15:36:46,942 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ad452c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:36:46,942 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7a780d55532c,44223,-1 for getting cluster id 2024-11-17T15:36:46,942 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T15:36:46,944 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e01f4cd2-8971-4a73-861a-2ca0deb4f76e' 2024-11-17T15:36:46,944 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T15:36:46,945 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e01f4cd2-8971-4a73-861a-2ca0deb4f76e" 2024-11-17T15:36:46,945 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f535689, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:36:46,945 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7a780d55532c,44223,-1] 2024-11-17T15:36:46,945 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T15:36:46,946 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:36:46,947 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41672, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T15:36:46,949 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10fbfe70, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:36:46,949 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T15:36:46,951 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7a780d55532c,32955,1731857800648, seqNum=-1] 2024-11-17T15:36:46,951 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T15:36:46,953 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57472, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T15:36:46,955 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-17T15:36:46,957 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 7a780d55532c,44223,1731857799978 2024-11-17T15:36:46,957 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@3dd53430 2024-11-17T15:36:46,959 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-17T15:36:46,961 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41688, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-17T15:36:46,965 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19d419ec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:36:46,965 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7a780d55532c,44223,-1 for getting cluster id 2024-11-17T15:36:46,965 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T15:36:46,966 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e01f4cd2-8971-4a73-861a-2ca0deb4f76e' 2024-11-17T15:36:46,967 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T15:36:46,967 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e01f4cd2-8971-4a73-861a-2ca0deb4f76e" 2024-11-17T15:36:46,967 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4fc5425e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:36:46,967 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7a780d55532c,44223,-1] 2024-11-17T15:36:46,968 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T15:36:46,968 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:36:46,969 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41702, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T15:36:46,970 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@18773756, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:36:46,971 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@130d4a2a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:36:46,971 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7a780d55532c,44223,-1 for getting cluster id 2024-11-17T15:36:46,971 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T15:36:46,972 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e01f4cd2-8971-4a73-861a-2ca0deb4f76e' 2024-11-17T15:36:46,973 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T15:36:46,973 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e01f4cd2-8971-4a73-861a-2ca0deb4f76e" 2024-11-17T15:36:46,973 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1eb99d17, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:36:46,973 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7a780d55532c,44223,-1] 2024-11-17T15:36:46,974 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T15:36:46,974 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:36:46,975 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41720, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T15:36:46,976 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7aafb42a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:36:46,977 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59d35085, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:36:46,977 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7a780d55532c,44223,-1 for getting cluster id 2024-11-17T15:36:46,977 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T15:36:46,979 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e01f4cd2-8971-4a73-861a-2ca0deb4f76e' 2024-11-17T15:36:46,980 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T15:36:46,980 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e01f4cd2-8971-4a73-861a-2ca0deb4f76e" 2024-11-17T15:36:46,980 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@546221b7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:36:46,980 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7a780d55532c,44223,-1] 2024-11-17T15:36:46,981 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T15:36:46,981 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:36:46,983 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41732, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T15:36:46,984 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c373c50, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:36:46,984 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fb4b46b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:36:46,984 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7a780d55532c,44223,-1 for getting cluster id 2024-11-17T15:36:46,985 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T15:36:46,986 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e01f4cd2-8971-4a73-861a-2ca0deb4f76e' 2024-11-17T15:36:46,986 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T15:36:46,986 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e01f4cd2-8971-4a73-861a-2ca0deb4f76e" 2024-11-17T15:36:46,987 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@551748ce, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:36:46,987 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7a780d55532c,44223,-1] 2024-11-17T15:36:46,987 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T15:36:46,988 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:36:46,989 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41752, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T15:36:46,990 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@192803e3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:36:46,990 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a1cce95, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:36:46,990 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7a780d55532c,44223,-1 for getting cluster id 2024-11-17T15:36:46,990 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T15:36:46,991 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e01f4cd2-8971-4a73-861a-2ca0deb4f76e' 2024-11-17T15:36:46,991 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T15:36:46,992 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e01f4cd2-8971-4a73-861a-2ca0deb4f76e" 2024-11-17T15:36:46,992 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49b6e9c5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:36:46,992 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7a780d55532c,44223,-1] 2024-11-17T15:36:46,992 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T15:36:46,992 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:36:46,993 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41770, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T15:36:46,994 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@699dc69a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:36:46,996 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3055dae7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:36:46,996 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7a780d55532c,44223,-1 for getting cluster id 2024-11-17T15:36:46,997 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T15:36:46,997 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e01f4cd2-8971-4a73-861a-2ca0deb4f76e' 2024-11-17T15:36:46,998 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T15:36:46,998 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e01f4cd2-8971-4a73-861a-2ca0deb4f76e" 2024-11-17T15:36:46,998 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e154f6d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:36:46,998 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7a780d55532c,44223,-1] 2024-11-17T15:36:46,998 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T15:36:46,999 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:36:47,000 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41786, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T15:36:47,000 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c8578f1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:36:47,002 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5238acdf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:36:47,003 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7a780d55532c,44223,-1 for getting cluster id 2024-11-17T15:36:47,003 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T15:36:47,004 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e01f4cd2-8971-4a73-861a-2ca0deb4f76e' 2024-11-17T15:36:47,004 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T15:36:47,004 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e01f4cd2-8971-4a73-861a-2ca0deb4f76e" 2024-11-17T15:36:47,004 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69746154, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:36:47,004 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7a780d55532c,44223,-1] 2024-11-17T15:36:47,005 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T15:36:47,005 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:36:47,006 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41802, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T15:36:47,007 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@354d66cc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:36:47,008 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f4f2c18, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:36:47,008 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7a780d55532c,44223,-1 for getting cluster id 2024-11-17T15:36:47,008 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T15:36:47,009 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e01f4cd2-8971-4a73-861a-2ca0deb4f76e' 2024-11-17T15:36:47,009 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T15:36:47,010 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e01f4cd2-8971-4a73-861a-2ca0deb4f76e" 2024-11-17T15:36:47,010 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35d66170, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:36:47,010 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7a780d55532c,44223,-1] 2024-11-17T15:36:47,010 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T15:36:47,010 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:36:47,011 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41822, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T15:36:47,012 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4bb69f80, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:36:47,013 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6643136e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:36:47,013 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7a780d55532c,44223,-1 for getting cluster id 2024-11-17T15:36:47,013 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T15:36:47,014 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e01f4cd2-8971-4a73-861a-2ca0deb4f76e' 2024-11-17T15:36:47,015 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T15:36:47,015 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e01f4cd2-8971-4a73-861a-2ca0deb4f76e" 2024-11-17T15:36:47,015 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d05f4b2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:36:47,015 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7a780d55532c,44223,-1] 2024-11-17T15:36:47,015 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T15:36:47,015 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:36:47,016 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41850, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T15:36:47,018 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d753fee, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:36:47,023 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T15:36:47,023 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T15:36:47,023 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T15:36:47,023 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T15:36:47,025 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7a780d55532c,32955,1731857800648, seqNum=-1] 2024-11-17T15:36:47,025 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T15:36:47,026 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T15:36:47,026 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7a780d55532c,32955,1731857800648, seqNum=-1] 2024-11-17T15:36:47,027 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7a780d55532c,32955,1731857800648, seqNum=-1] 2024-11-17T15:36:47,027 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T15:36:47,027 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T15:36:47,029 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57488, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T15:36:47,029 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57496, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T15:36:47,029 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57502, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T15:36:47,029 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7a780d55532c,32955,1731857800648, seqNum=-1] 2024-11-17T15:36:47,030 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7a780d55532c,32955,1731857800648, seqNum=-1] 2024-11-17T15:36:47,030 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T15:36:47,030 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T15:36:47,032 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57506, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T15:36:47,032 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57522, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T15:36:47,033 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T15:36:47,034 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T15:36:47,037 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2] 2024-11-17T15:36:47,041 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T15:36:47,041 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7a780d55532c,32955,1731857800648, seqNum=-1] 2024-11-17T15:36:47,042 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T15:36:47,042 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2] 2024-11-17T15:36:47,043 DEBUG [Thread-151 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7a780d55532c,32955,1731857800648, seqNum=-1] 2024-11-17T15:36:47,044 DEBUG [Thread-151 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T15:36:47,044 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2] 2024-11-17T15:36:47,046 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7a780d55532c,32955,1731857800648, seqNum=-1] 2024-11-17T15:36:47,046 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57530, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T15:36:47,047 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T15:36:47,047 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2] 2024-11-17T15:36:47,047 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57532, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T15:36:47,049 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57540, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T15:36:47,049 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T15:36:47,050 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2] 2024-11-17T15:36:47,054 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7a780d55532c,32955,1731857800648, seqNum=-1] 2024-11-17T15:36:47,054 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2] 2024-11-17T15:36:47,054 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2] 2024-11-17T15:36:47,054 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T15:36:47,056 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2] 2024-11-17T15:36:47,058 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57542, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T15:36:47,061 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2] 2024-11-17T15:36:47,070 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T15:36:47,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-11-17T15:36:47,078 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T15:36:47,081 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T15:36:47,082 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T15:36:47,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-17T15:36:47,147 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T15:36:47,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(8855): Flush requested on a80c4e623aab44bea3cfa33ee4fee8f3 2024-11-17T15:36:47,152 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a80c4e623aab44bea3cfa33ee4fee8f3 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-17T15:36:47,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-17T15:36:47,271 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-17T15:36:47,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3. 2024-11-17T15:36:47,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3. as already flushing 2024-11-17T15:36:47,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3. 2024-11-17T15:36:47,295 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => a80c4e623aab44bea3cfa33ee4fee8f3, NAME => 'TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:36:47,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => a80c4e623aab44bea3cfa33ee4fee8f3, NAME => 'TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:36:47,316 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/A/d097e30ef8c34ce6a5c8e59e8f175f5a is 50, key is test_row_0/A:col10/1731857807142/Put/seqid=0 2024-11-17T15:36:47,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a80c4e623aab44bea3cfa33ee4fee8f3, NAME => 'TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a80c4e623aab44bea3cfa33ee4fee8f3, NAME => 'TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:36:47,346 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:36:47,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57522 deadline: 1731857867334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 2024-11-17T15:36:47,352 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:36:47,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57502 deadline: 1731857867337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 2024-11-17T15:36:47,354 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:47,354 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:47,354 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:47,354 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:47,355 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:36:47,355 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:36:47,360 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:36:47,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57530 deadline: 1731857867346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 2024-11-17T15:36:47,361 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:47,361 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:47,361 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:36:47,363 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:36:47,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57488 deadline: 1731857867347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 2024-11-17T15:36:47,365 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:36:47,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57496 deadline: 1731857867351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 2024-11-17T15:36:47,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073741837_1013 (size=14341) 2024-11-17T15:36:47,366 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:47,366 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:47,366 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:47,366 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:36:47,367 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:47,367 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:36:47,367 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/A/d097e30ef8c34ce6a5c8e59e8f175f5a 2024-11-17T15:36:47,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-17T15:36:47,477 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:36:47,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57530 deadline: 1731857867466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 2024-11-17T15:36:47,478 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-17T15:36:47,479 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:36:47,479 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:47,479 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:47,479 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:36:47,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57522 deadline: 1731857867466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 2024-11-17T15:36:47,480 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:47,480 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:47,480 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:36:47,480 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:36:47,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57502 deadline: 1731857867466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 2024-11-17T15:36:47,481 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:47,482 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:47,482 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:36:47,482 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:36:47,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57488 deadline: 1731857867478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 2024-11-17T15:36:47,483 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:47,483 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:47,484 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:36:47,486 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:36:47,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57496 deadline: 1731857867485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 2024-11-17T15:36:47,487 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:47,487 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:47,487 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:36:47,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3. 2024-11-17T15:36:47,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3. as already flushing 2024-11-17T15:36:47,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3. 2024-11-17T15:36:47,493 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => a80c4e623aab44bea3cfa33ee4fee8f3, NAME => 'TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:36:47,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => a80c4e623aab44bea3cfa33ee4fee8f3, NAME => 'TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:36:47,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a80c4e623aab44bea3cfa33ee4fee8f3, NAME => 'TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a80c4e623aab44bea3cfa33ee4fee8f3, NAME => 'TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:36:47,509 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/B/73bd1efe107344cfb38a8802ed6648a3 is 50, key is test_row_0/B:col10/1731857807142/Put/seqid=0 2024-11-17T15:36:47,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073741838_1014 (size=12001) 2024-11-17T15:36:47,654 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-17T15:36:47,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3. 2024-11-17T15:36:47,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3. as already flushing 2024-11-17T15:36:47,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3. 2024-11-17T15:36:47,657 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => a80c4e623aab44bea3cfa33ee4fee8f3, NAME => 'TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:36:47,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => a80c4e623aab44bea3cfa33ee4fee8f3, NAME => 'TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:36:47,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a80c4e623aab44bea3cfa33ee4fee8f3, NAME => 'TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a80c4e623aab44bea3cfa33ee4fee8f3, NAME => 'TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:36:47,693 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:36:47,693 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:36:47,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57530 deadline: 1731857867684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 2024-11-17T15:36:47,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57522 deadline: 1731857867685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 2024-11-17T15:36:47,694 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:47,694 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:47,694 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:36:47,696 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:47,696 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:47,696 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:36:47,697 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:36:47,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57496 deadline: 1731857867695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 2024-11-17T15:36:47,698 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:36:47,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57488 deadline: 1731857867695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 2024-11-17T15:36:47,699 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:36:47,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57502 deadline: 1731857867696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 2024-11-17T15:36:47,700 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:47,700 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:47,700 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:36:47,700 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:47,701 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:47,701 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:36:47,702 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:47,702 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:47,702 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:36:47,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-17T15:36:47,813 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-17T15:36:47,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3. 2024-11-17T15:36:47,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3. as already flushing 2024-11-17T15:36:47,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3. 2024-11-17T15:36:47,815 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => a80c4e623aab44bea3cfa33ee4fee8f3, NAME => 'TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:36:47,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => a80c4e623aab44bea3cfa33ee4fee8f3, NAME => 'TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:36:47,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a80c4e623aab44bea3cfa33ee4fee8f3, NAME => 'TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a80c4e623aab44bea3cfa33ee4fee8f3, NAME => 'TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:36:47,969 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/B/73bd1efe107344cfb38a8802ed6648a3 2024-11-17T15:36:47,971 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-17T15:36:47,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3. 2024-11-17T15:36:47,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3. as already flushing 2024-11-17T15:36:47,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3. 2024-11-17T15:36:47,973 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => a80c4e623aab44bea3cfa33ee4fee8f3, NAME => 'TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:36:47,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => a80c4e623aab44bea3cfa33ee4fee8f3, NAME => 'TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:36:47,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a80c4e623aab44bea3cfa33ee4fee8f3, NAME => 'TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a80c4e623aab44bea3cfa33ee4fee8f3, NAME => 'TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:36:48,006 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:36:48,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57530 deadline: 1731857868004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 2024-11-17T15:36:48,008 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:36:48,008 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:48,008 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:48,008 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:36:48,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57488 deadline: 1731857868005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 2024-11-17T15:36:48,009 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:48,009 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:48,009 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:36:48,009 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:36:48,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57522 deadline: 1731857868005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 2024-11-17T15:36:48,010 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:48,010 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:48,010 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:36:48,013 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/C/543e81de5f5d404dbf686651a01cf73f is 50, key is test_row_0/C:col10/1731857807142/Put/seqid=0 2024-11-17T15:36:48,017 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:36:48,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57502 deadline: 1731857868014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 2024-11-17T15:36:48,017 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:36:48,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57496 deadline: 1731857868015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 2024-11-17T15:36:48,018 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:48,018 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:48,018 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:36:48,020 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:48,020 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:48,020 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:36:48,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073741839_1015 (size=12001) 2024-11-17T15:36:48,055 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/C/543e81de5f5d404dbf686651a01cf73f 2024-11-17T15:36:48,073 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/A/d097e30ef8c34ce6a5c8e59e8f175f5a as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/A/d097e30ef8c34ce6a5c8e59e8f175f5a 2024-11-17T15:36:48,085 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/A/d097e30ef8c34ce6a5c8e59e8f175f5a, entries=200, sequenceid=14, filesize=14.0 K 2024-11-17T15:36:48,092 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/B/73bd1efe107344cfb38a8802ed6648a3 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/B/73bd1efe107344cfb38a8802ed6648a3 2024-11-17T15:36:48,111 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/B/73bd1efe107344cfb38a8802ed6648a3, entries=150, sequenceid=14, filesize=11.7 K 2024-11-17T15:36:48,113 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/C/543e81de5f5d404dbf686651a01cf73f as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/C/543e81de5f5d404dbf686651a01cf73f 2024-11-17T15:36:48,128 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/C/543e81de5f5d404dbf686651a01cf73f, entries=150, sequenceid=14, filesize=11.7 K 2024-11-17T15:36:48,132 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-17T15:36:48,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3. 2024-11-17T15:36:48,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3. as already flushing 2024-11-17T15:36:48,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3. 2024-11-17T15:36:48,133 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => a80c4e623aab44bea3cfa33ee4fee8f3, NAME => 'TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:36:48,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => a80c4e623aab44bea3cfa33ee4fee8f3, NAME => 'TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:36:48,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a80c4e623aab44bea3cfa33ee4fee8f3, NAME => 'TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a80c4e623aab44bea3cfa33ee4fee8f3, NAME => 'TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:36:48,137 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for a80c4e623aab44bea3cfa33ee4fee8f3 in 984ms, sequenceid=14, compaction requested=false 2024-11-17T15:36:48,139 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-17T15:36:48,141 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a80c4e623aab44bea3cfa33ee4fee8f3: 2024-11-17T15:36:48,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-17T15:36:48,289 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-17T15:36:48,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3. 2024-11-17T15:36:48,290 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing a80c4e623aab44bea3cfa33ee4fee8f3 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-17T15:36:48,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/A/38865204207448788c9dd4870cc1e94b is 50, key is test_row_0/A:col10/1731857807332/Put/seqid=0 2024-11-17T15:36:48,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073741840_1016 (size=12001) 2024-11-17T15:36:48,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(8855): Flush requested on a80c4e623aab44bea3cfa33ee4fee8f3 2024-11-17T15:36:48,522 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3. as already flushing 2024-11-17T15:36:48,549 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:36:48,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57530 deadline: 1731857868536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 2024-11-17T15:36:48,550 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:48,550 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:48,550 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:36:48,550 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:36:48,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57488 deadline: 1731857868542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 2024-11-17T15:36:48,551 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:48,551 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:36:48,552 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:48,552 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:36:48,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57522 deadline: 1731857868544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 2024-11-17T15:36:48,552 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:48,552 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:48,552 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:36:48,563 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:36:48,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57496 deadline: 1731857868551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 2024-11-17T15:36:48,564 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:48,564 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:48,564 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:36:48,565 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:36:48,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57502 deadline: 1731857868558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 2024-11-17T15:36:48,566 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:48,566 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:48,566 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:36:48,654 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:36:48,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57530 deadline: 1731857868653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 2024-11-17T15:36:48,656 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:48,656 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:48,656 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:36:48,666 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:36:48,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57488 deadline: 1731857868664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 2024-11-17T15:36:48,667 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:48,667 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:48,668 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:36:48,668 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:36:48,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57522 deadline: 1731857868664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 2024-11-17T15:36:48,669 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:48,670 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:48,670 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:36:48,676 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:36:48,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57502 deadline: 1731857868674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 2024-11-17T15:36:48,678 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:48,678 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:48,678 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:36:48,686 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:36:48,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57496 deadline: 1731857868676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 2024-11-17T15:36:48,687 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:48,687 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:48,687 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:36:48,761 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/A/38865204207448788c9dd4870cc1e94b 2024-11-17T15:36:48,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/B/effa15d312364f1f9fa6183342b6b378 is 50, key is test_row_0/B:col10/1731857807332/Put/seqid=0 2024-11-17T15:36:48,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073741841_1017 (size=12001) 2024-11-17T15:36:48,814 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/B/effa15d312364f1f9fa6183342b6b378 2024-11-17T15:36:48,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/C/460d6067d4d04b5e9ceacf1b12789ddc is 50, key is test_row_0/C:col10/1731857807332/Put/seqid=0 2024-11-17T15:36:48,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073741842_1018 (size=12001) 2024-11-17T15:36:48,871 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:36:48,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57530 deadline: 1731857868865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 2024-11-17T15:36:48,873 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:48,873 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:48,873 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:36:48,875 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:36:48,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57522 deadline: 1731857868874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 2024-11-17T15:36:48,875 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:48,876 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:48,876 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:36:48,876 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:36:48,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57488 deadline: 1731857868874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 2024-11-17T15:36:48,878 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:48,878 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:48,878 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:36:48,887 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:36:48,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57502 deadline: 1731857868883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 2024-11-17T15:36:48,888 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:48,888 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:48,888 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:36:48,895 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:36:48,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57496 deadline: 1731857868894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 2024-11-17T15:36:48,895 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:48,896 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:48,896 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:36:49,140 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-17T15:36:49,191 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:36:49,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57522 deadline: 1731857869184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 2024-11-17T15:36:49,192 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:49,192 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:49,192 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:36:49,192 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:36:49,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57488 deadline: 1731857869185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 2024-11-17T15:36:49,193 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:36:49,194 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:49,194 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:49,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57530 deadline: 1731857869184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 2024-11-17T15:36:49,194 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:36:49,195 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:49,195 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:49,195 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:36:49,197 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:36:49,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57502 deadline: 1731857869194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 2024-11-17T15:36:49,198 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:49,198 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:49,198 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:36:49,208 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:36:49,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57496 deadline: 1731857869204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 2024-11-17T15:36:49,209 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:49,209 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:49,209 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:36:49,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-17T15:36:49,272 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/C/460d6067d4d04b5e9ceacf1b12789ddc 2024-11-17T15:36:49,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/A/38865204207448788c9dd4870cc1e94b as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/A/38865204207448788c9dd4870cc1e94b 2024-11-17T15:36:49,306 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/A/38865204207448788c9dd4870cc1e94b, entries=150, sequenceid=37, filesize=11.7 K 2024-11-17T15:36:49,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/B/effa15d312364f1f9fa6183342b6b378 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/B/effa15d312364f1f9fa6183342b6b378 2024-11-17T15:36:49,331 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/B/effa15d312364f1f9fa6183342b6b378, entries=150, sequenceid=37, filesize=11.7 K 2024-11-17T15:36:49,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/C/460d6067d4d04b5e9ceacf1b12789ddc as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/C/460d6067d4d04b5e9ceacf1b12789ddc 2024-11-17T15:36:49,360 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/C/460d6067d4d04b5e9ceacf1b12789ddc, entries=150, sequenceid=37, filesize=11.7 K 2024-11-17T15:36:49,364 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for a80c4e623aab44bea3cfa33ee4fee8f3 in 1073ms, sequenceid=37, compaction requested=false 2024-11-17T15:36:49,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for a80c4e623aab44bea3cfa33ee4fee8f3: 2024-11-17T15:36:49,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3. 2024-11-17T15:36:49,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-11-17T15:36:49,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-11-17T15:36:49,379 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-17T15:36:49,379 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2850 sec 2024-11-17T15:36:49,387 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 2.3090 sec 2024-11-17T15:36:49,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(8855): Flush requested on a80c4e623aab44bea3cfa33ee4fee8f3 2024-11-17T15:36:49,711 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T15:36:49,711 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a80c4e623aab44bea3cfa33ee4fee8f3 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-17T15:36:49,731 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/A/19b314941ee246df9bc8cb722efe87f2 is 50, key is test_row_0/A:col10/1731857808532/Put/seqid=0 2024-11-17T15:36:49,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073741843_1019 (size=14341) 2024-11-17T15:36:49,771 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/A/19b314941ee246df9bc8cb722efe87f2 2024-11-17T15:36:49,781 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:36:49,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57496 deadline: 1731857869765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 2024-11-17T15:36:49,783 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:49,783 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:36:49,783 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:49,783 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:36:49,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57488 deadline: 1731857869771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 2024-11-17T15:36:49,784 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:49,784 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:36:49,784 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:49,784 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:36:49,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57530 deadline: 1731857869774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 2024-11-17T15:36:49,785 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:49,785 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:49,785 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:36:49,785 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:36:49,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57502 deadline: 1731857869776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 2024-11-17T15:36:49,786 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:49,786 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:49,786 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:36:49,786 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:36:49,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57522 deadline: 1731857869781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 2024-11-17T15:36:49,787 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:49,787 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:49,787 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:36:49,799 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/B/a88b22ae672744e884bc4a551099ab62 is 50, key is test_row_0/B:col10/1731857808532/Put/seqid=0 2024-11-17T15:36:49,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073741844_1020 (size=12001) 2024-11-17T15:36:49,833 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/B/a88b22ae672744e884bc4a551099ab62 2024-11-17T15:36:49,855 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-17T15:36:49,867 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/C/dc66268b3dcc4035a5087570dc0f411a is 50, key is test_row_0/C:col10/1731857808532/Put/seqid=0 2024-11-17T15:36:49,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073741845_1021 (size=12001) 2024-11-17T15:36:49,884 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/C/dc66268b3dcc4035a5087570dc0f411a 2024-11-17T15:36:49,900 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:36:49,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57488 deadline: 1731857869894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 2024-11-17T15:36:49,902 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:36:49,902 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:36:49,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57530 deadline: 1731857869894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 2024-11-17T15:36:49,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57522 deadline: 1731857869894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 2024-11-17T15:36:49,903 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:49,903 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:49,903 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:49,903 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:49,903 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:36:49,903 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:36:49,904 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:49,904 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:49,904 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:36:49,905 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:36:49,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57496 deadline: 1731857869895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 2024-11-17T15:36:49,906 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:49,906 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:49,906 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:36:49,906 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:36:49,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57502 deadline: 1731857869903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 2024-11-17T15:36:49,907 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:49,907 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:49,907 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:36:49,913 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/A/19b314941ee246df9bc8cb722efe87f2 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/A/19b314941ee246df9bc8cb722efe87f2 2024-11-17T15:36:49,927 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/A/19b314941ee246df9bc8cb722efe87f2, entries=200, sequenceid=51, filesize=14.0 K 2024-11-17T15:36:49,929 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/B/a88b22ae672744e884bc4a551099ab62 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/B/a88b22ae672744e884bc4a551099ab62 2024-11-17T15:36:49,950 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/B/a88b22ae672744e884bc4a551099ab62, entries=150, sequenceid=51, filesize=11.7 K 2024-11-17T15:36:49,952 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/C/dc66268b3dcc4035a5087570dc0f411a as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/C/dc66268b3dcc4035a5087570dc0f411a 2024-11-17T15:36:49,964 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/C/dc66268b3dcc4035a5087570dc0f411a, entries=150, sequenceid=51, filesize=11.7 K 2024-11-17T15:36:49,967 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for a80c4e623aab44bea3cfa33ee4fee8f3 in 255ms, sequenceid=51, compaction requested=true 2024-11-17T15:36:49,967 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a80c4e623aab44bea3cfa33ee4fee8f3: 2024-11-17T15:36:49,975 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a80c4e623aab44bea3cfa33ee4fee8f3:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T15:36:49,978 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T15:36:49,978 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a80c4e623aab44bea3cfa33ee4fee8f3:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T15:36:49,978 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T15:36:49,978 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a80c4e623aab44bea3cfa33ee4fee8f3:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T15:36:49,978 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T15:36:49,981 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T15:36:49,982 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T15:36:49,985 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T15:36:49,987 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1541): a80c4e623aab44bea3cfa33ee4fee8f3/B is initiating minor compaction (all files) 2024-11-17T15:36:49,987 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a80c4e623aab44bea3cfa33ee4fee8f3/B in TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3. 2024-11-17T15:36:49,988 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/B/73bd1efe107344cfb38a8802ed6648a3, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/B/effa15d312364f1f9fa6183342b6b378, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/B/a88b22ae672744e884bc4a551099ab62] into tmpdir=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp, totalSize=35.2 K 2024-11-17T15:36:49,990 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting 73bd1efe107344cfb38a8802ed6648a3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731857807081 2024-11-17T15:36:49,991 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting effa15d312364f1f9fa6183342b6b378, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731857807299 2024-11-17T15:36:49,991 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40683 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T15:36:49,991 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1541): a80c4e623aab44bea3cfa33ee4fee8f3/A is initiating minor compaction (all files) 2024-11-17T15:36:49,992 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a80c4e623aab44bea3cfa33ee4fee8f3/A in TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3. 2024-11-17T15:36:49,992 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/A/d097e30ef8c34ce6a5c8e59e8f175f5a, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/A/38865204207448788c9dd4870cc1e94b, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/A/19b314941ee246df9bc8cb722efe87f2] into tmpdir=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp, totalSize=39.7 K 2024-11-17T15:36:49,992 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting a88b22ae672744e884bc4a551099ab62, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731857808532 2024-11-17T15:36:49,993 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting d097e30ef8c34ce6a5c8e59e8f175f5a, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731857807081 2024-11-17T15:36:49,994 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 38865204207448788c9dd4870cc1e94b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731857807299 2024-11-17T15:36:49,995 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 19b314941ee246df9bc8cb722efe87f2, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731857808532 2024-11-17T15:36:50,039 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a80c4e623aab44bea3cfa33ee4fee8f3#B#compaction#9 average throughput is 0.39 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T15:36:50,041 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/B/54f3a827533347dc8644ec6bb313dae9 is 50, key is test_row_0/B:col10/1731857808532/Put/seqid=0 2024-11-17T15:36:50,063 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a80c4e623aab44bea3cfa33ee4fee8f3#A#compaction#10 average throughput is 0.60 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T15:36:50,064 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/A/bf4940d3105d4ea09ed03e1888157d51 is 50, key is test_row_0/A:col10/1731857808532/Put/seqid=0 2024-11-17T15:36:50,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073741846_1022 (size=12104) 2024-11-17T15:36:50,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073741847_1023 (size=12104) 2024-11-17T15:36:50,092 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/B/54f3a827533347dc8644ec6bb313dae9 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/B/54f3a827533347dc8644ec6bb313dae9 2024-11-17T15:36:50,095 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/A/bf4940d3105d4ea09ed03e1888157d51 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/A/bf4940d3105d4ea09ed03e1888157d51 2024-11-17T15:36:50,126 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a80c4e623aab44bea3cfa33ee4fee8f3/A of a80c4e623aab44bea3cfa33ee4fee8f3 into bf4940d3105d4ea09ed03e1888157d51(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T15:36:50,127 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a80c4e623aab44bea3cfa33ee4fee8f3: 2024-11-17T15:36:50,127 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a80c4e623aab44bea3cfa33ee4fee8f3/B of a80c4e623aab44bea3cfa33ee4fee8f3 into 54f3a827533347dc8644ec6bb313dae9(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T15:36:50,127 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a80c4e623aab44bea3cfa33ee4fee8f3: 2024-11-17T15:36:50,130 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., storeName=a80c4e623aab44bea3cfa33ee4fee8f3/A, priority=13, startTime=1731857809968; duration=0sec 2024-11-17T15:36:50,130 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., storeName=a80c4e623aab44bea3cfa33ee4fee8f3/B, priority=13, startTime=1731857809978; duration=0sec 2024-11-17T15:36:50,130 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T15:36:50,130 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T15:36:50,130 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a80c4e623aab44bea3cfa33ee4fee8f3:B 2024-11-17T15:36:50,130 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a80c4e623aab44bea3cfa33ee4fee8f3:A 2024-11-17T15:36:50,131 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T15:36:50,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(8855): Flush requested on a80c4e623aab44bea3cfa33ee4fee8f3 2024-11-17T15:36:50,132 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T15:36:50,133 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a80c4e623aab44bea3cfa33ee4fee8f3 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-17T15:36:50,144 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/A/f4855d7654e548969ed0272bd95df179 is 50, key is test_row_0/A:col10/1731857809777/Put/seqid=0 2024-11-17T15:36:50,146 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T15:36:50,147 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1541): a80c4e623aab44bea3cfa33ee4fee8f3/C is initiating minor compaction (all files) 2024-11-17T15:36:50,147 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a80c4e623aab44bea3cfa33ee4fee8f3/C in TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3. 2024-11-17T15:36:50,147 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/C/543e81de5f5d404dbf686651a01cf73f, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/C/460d6067d4d04b5e9ceacf1b12789ddc, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/C/dc66268b3dcc4035a5087570dc0f411a] into tmpdir=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp, totalSize=35.2 K 2024-11-17T15:36:50,148 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 543e81de5f5d404dbf686651a01cf73f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731857807081 2024-11-17T15:36:50,149 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 460d6067d4d04b5e9ceacf1b12789ddc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731857807299 2024-11-17T15:36:50,150 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting dc66268b3dcc4035a5087570dc0f411a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731857808532 2024-11-17T15:36:50,187 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073741848_1024 (size=12001) 2024-11-17T15:36:50,189 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/A/f4855d7654e548969ed0272bd95df179 2024-11-17T15:36:50,194 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:36:50,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57496 deadline: 1731857870175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 2024-11-17T15:36:50,195 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:36:50,195 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:50,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57502 deadline: 1731857870171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 2024-11-17T15:36:50,195 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:50,196 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:36:50,196 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:50,196 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:50,196 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:36:50,211 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:36:50,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57488 deadline: 1731857870195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 2024-11-17T15:36:50,212 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:50,212 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:50,212 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:36:50,213 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a80c4e623aab44bea3cfa33ee4fee8f3#C#compaction#12 average throughput is 0.73 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T15:36:50,213 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:36:50,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57530 deadline: 1731857870196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 2024-11-17T15:36:50,214 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:50,214 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:50,214 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:36:50,214 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:36:50,214 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/C/dd28e9fd8ba74381868490f561a28208 is 50, key is test_row_0/C:col10/1731857808532/Put/seqid=0 2024-11-17T15:36:50,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57522 deadline: 1731857870198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 2024-11-17T15:36:50,215 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:50,215 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:50,215 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:36:50,226 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/B/173babc1b3494a33bd587d631d8d1e14 is 50, key is test_row_0/B:col10/1731857809777/Put/seqid=0 2024-11-17T15:36:50,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073741849_1025 (size=12104) 2024-11-17T15:36:50,265 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073741850_1026 (size=12001) 2024-11-17T15:36:50,268 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/C/dd28e9fd8ba74381868490f561a28208 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/C/dd28e9fd8ba74381868490f561a28208 2024-11-17T15:36:50,285 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a80c4e623aab44bea3cfa33ee4fee8f3/C of a80c4e623aab44bea3cfa33ee4fee8f3 into dd28e9fd8ba74381868490f561a28208(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T15:36:50,286 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a80c4e623aab44bea3cfa33ee4fee8f3: 2024-11-17T15:36:50,286 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., storeName=a80c4e623aab44bea3cfa33ee4fee8f3/C, priority=13, startTime=1731857809978; duration=0sec 2024-11-17T15:36:50,286 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T15:36:50,286 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a80c4e623aab44bea3cfa33ee4fee8f3:C 2024-11-17T15:36:50,313 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:36:50,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57496 deadline: 1731857870307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 2024-11-17T15:36:50,314 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:36:50,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57502 deadline: 1731857870307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 2024-11-17T15:36:50,315 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:50,315 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:50,315 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:50,315 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:50,315 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:36:50,315 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:36:50,329 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:36:50,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57530 deadline: 1731857870327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 2024-11-17T15:36:50,331 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:50,331 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:50,331 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:36:50,331 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:36:50,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57522 deadline: 1731857870327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 2024-11-17T15:36:50,332 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:50,332 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:50,332 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:36:50,332 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:36:50,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57488 deadline: 1731857870327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 2024-11-17T15:36:50,333 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:50,333 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:50,333 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:36:50,383 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-17T15:36:50,384 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-17T15:36:50,386 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-17T15:36:50,386 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-17T15:36:50,387 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-17T15:36:50,387 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-17T15:36:50,388 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-17T15:36:50,388 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-17T15:36:50,535 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:36:50,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57502 deadline: 1731857870524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 2024-11-17T15:36:50,538 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:50,539 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:50,539 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:36:50,541 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:36:50,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57496 deadline: 1731857870525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 2024-11-17T15:36:50,542 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:50,542 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:50,542 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:36:50,548 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:36:50,548 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:36:50,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57522 deadline: 1731857870544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 2024-11-17T15:36:50,549 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:50,549 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:50,549 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:36:50,549 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:36:50,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57530 deadline: 1731857870545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 2024-11-17T15:36:50,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57488 deadline: 1731857870544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 2024-11-17T15:36:50,550 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:50,550 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:50,551 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:36:50,551 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:50,551 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:50,551 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:36:50,667 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/B/173babc1b3494a33bd587d631d8d1e14 2024-11-17T15:36:50,704 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/C/ce9ced83164e42da9aa5e6c22eb49366 is 50, key is test_row_0/C:col10/1731857809777/Put/seqid=0 2024-11-17T15:36:50,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073741851_1027 (size=12001) 2024-11-17T15:36:50,743 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/C/ce9ced83164e42da9aa5e6c22eb49366 2024-11-17T15:36:50,758 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/A/f4855d7654e548969ed0272bd95df179 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/A/f4855d7654e548969ed0272bd95df179 2024-11-17T15:36:50,781 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/A/f4855d7654e548969ed0272bd95df179, entries=150, sequenceid=77, filesize=11.7 K 2024-11-17T15:36:50,785 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/B/173babc1b3494a33bd587d631d8d1e14 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/B/173babc1b3494a33bd587d631d8d1e14 2024-11-17T15:36:50,799 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/B/173babc1b3494a33bd587d631d8d1e14, entries=150, sequenceid=77, filesize=11.7 K 2024-11-17T15:36:50,802 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/C/ce9ced83164e42da9aa5e6c22eb49366 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/C/ce9ced83164e42da9aa5e6c22eb49366 2024-11-17T15:36:50,828 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/C/ce9ced83164e42da9aa5e6c22eb49366, entries=150, sequenceid=77, filesize=11.7 K 2024-11-17T15:36:50,830 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=73.80 KB/75570 for a80c4e623aab44bea3cfa33ee4fee8f3 in 697ms, sequenceid=77, compaction requested=false 2024-11-17T15:36:50,830 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a80c4e623aab44bea3cfa33ee4fee8f3: 2024-11-17T15:36:50,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(8855): Flush requested on a80c4e623aab44bea3cfa33ee4fee8f3 2024-11-17T15:36:50,849 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T15:36:50,849 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a80c4e623aab44bea3cfa33ee4fee8f3 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-17T15:36:50,866 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/A/9a30184341e84c8f8a67425ab889451d is 50, key is test_row_0/A:col10/1731857810846/Put/seqid=0 2024-11-17T15:36:50,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073741852_1028 (size=12001) 2024-11-17T15:36:50,921 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:36:50,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57530 deadline: 1731857870902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 2024-11-17T15:36:50,922 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:36:50,922 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:50,922 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:36:50,922 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:50,923 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:36:50,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57488 deadline: 1731857870907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 2024-11-17T15:36:50,923 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:36:50,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57496 deadline: 1731857870910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 2024-11-17T15:36:50,924 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:50,924 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:50,924 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:36:50,926 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:50,926 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:50,926 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:36:50,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57502 deadline: 1731857870904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 2024-11-17T15:36:50,926 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:50,927 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:50,927 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:36:50,927 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:36:50,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57522 deadline: 1731857870921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 2024-11-17T15:36:50,928 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:50,928 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:50,928 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:36:51,040 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:36:51,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57522 deadline: 1731857871035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 2024-11-17T15:36:51,041 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:51,041 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:51,041 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:36:51,042 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:36:51,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57530 deadline: 1731857871035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 2024-11-17T15:36:51,043 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:36:51,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57488 deadline: 1731857871035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 2024-11-17T15:36:51,043 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:51,043 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:51,044 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:36:51,044 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:51,044 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:51,044 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:36:51,044 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:36:51,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57502 deadline: 1731857871036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 2024-11-17T15:36:51,042 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:36:51,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57496 deadline: 1731857871035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 2024-11-17T15:36:51,050 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:51,050 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:51,050 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:36:51,050 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:51,050 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:51,051 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:36:51,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-17T15:36:51,254 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T15:36:51,255 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T15:36:51,255 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:36:51,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57522 deadline: 1731857871254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 2024-11-17T15:36:51,256 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:36:51,256 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:51,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57488 deadline: 1731857871254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 2024-11-17T15:36:51,256 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:51,256 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:36:51,257 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:36:51,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-11-17T15:36:51,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57496 deadline: 1731857871255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 2024-11-17T15:36:51,257 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:36:51,258 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:51,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57530 deadline: 1731857871255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 2024-11-17T15:36:51,258 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:51,258 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:36:51,258 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:51,258 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:51,258 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:36:51,259 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:51,259 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:51,259 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:36:51,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-17T15:36:51,265 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:36:51,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57502 deadline: 1731857871264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 2024-11-17T15:36:51,266 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:51,266 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:51,266 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:36:51,268 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T15:36:51,272 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T15:36:51,272 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T15:36:51,319 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/A/9a30184341e84c8f8a67425ab889451d 2024-11-17T15:36:51,341 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/B/7dacf2ef7dd947d2ad37db118ff60ac5 is 50, key is test_row_0/B:col10/1731857810846/Put/seqid=0 2024-11-17T15:36:51,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-17T15:36:51,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073741853_1029 (size=12001) 2024-11-17T15:36:51,384 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/B/7dacf2ef7dd947d2ad37db118ff60ac5 2024-11-17T15:36:51,428 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-17T15:36:51,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3. 2024-11-17T15:36:51,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3. as already flushing 2024-11-17T15:36:51,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3. 2024-11-17T15:36:51,429 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => a80c4e623aab44bea3cfa33ee4fee8f3, NAME => 'TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:36:51,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => a80c4e623aab44bea3cfa33ee4fee8f3, NAME => 'TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:36:51,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a80c4e623aab44bea3cfa33ee4fee8f3, NAME => 'TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a80c4e623aab44bea3cfa33ee4fee8f3, NAME => 'TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:36:51,435 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/C/3cfef1ced9b3411fad18d6b66b84dca0 is 50, key is test_row_0/C:col10/1731857810846/Put/seqid=0 2024-11-17T15:36:51,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073741854_1030 (size=12001) 2024-11-17T15:36:51,568 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:36:51,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57530 deadline: 1731857871563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 2024-11-17T15:36:51,569 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:36:51,570 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:36:51,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57488 deadline: 1731857871564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 2024-11-17T15:36:51,570 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:51,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57522 deadline: 1731857871564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 2024-11-17T15:36:51,570 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:51,570 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:36:51,570 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:36:51,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57496 deadline: 1731857871564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 2024-11-17T15:36:51,572 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:51,572 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:51,572 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:36:51,573 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:51,573 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:51,573 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:36:51,574 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:51,574 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:51,574 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:36:51,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-17T15:36:51,575 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:36:51,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57502 deadline: 1731857871574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 2024-11-17T15:36:51,575 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:51,576 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:51,576 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:36:51,584 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-17T15:36:51,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3. 2024-11-17T15:36:51,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3. as already flushing 2024-11-17T15:36:51,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3. 2024-11-17T15:36:51,585 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => a80c4e623aab44bea3cfa33ee4fee8f3, NAME => 'TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:36:51,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => a80c4e623aab44bea3cfa33ee4fee8f3, NAME => 'TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:36:51,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a80c4e623aab44bea3cfa33ee4fee8f3, NAME => 'TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a80c4e623aab44bea3cfa33ee4fee8f3, NAME => 'TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:36:51,745 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-17T15:36:51,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3. 2024-11-17T15:36:51,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3. as already flushing 2024-11-17T15:36:51,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3. 2024-11-17T15:36:51,746 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => a80c4e623aab44bea3cfa33ee4fee8f3, NAME => 'TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:36:51,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => a80c4e623aab44bea3cfa33ee4fee8f3, NAME => 'TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:36:51,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a80c4e623aab44bea3cfa33ee4fee8f3, NAME => 'TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a80c4e623aab44bea3cfa33ee4fee8f3, NAME => 'TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:36:51,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-17T15:36:51,896 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/C/3cfef1ced9b3411fad18d6b66b84dca0 2024-11-17T15:36:51,902 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-17T15:36:51,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3. 2024-11-17T15:36:51,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3. as already flushing 2024-11-17T15:36:51,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3. 2024-11-17T15:36:51,903 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => a80c4e623aab44bea3cfa33ee4fee8f3, NAME => 'TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:36:51,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => a80c4e623aab44bea3cfa33ee4fee8f3, NAME => 'TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:36:51,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a80c4e623aab44bea3cfa33ee4fee8f3, NAME => 'TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a80c4e623aab44bea3cfa33ee4fee8f3, NAME => 'TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:36:51,924 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/A/9a30184341e84c8f8a67425ab889451d as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/A/9a30184341e84c8f8a67425ab889451d 2024-11-17T15:36:51,946 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/A/9a30184341e84c8f8a67425ab889451d, entries=150, sequenceid=93, filesize=11.7 K 2024-11-17T15:36:51,948 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/B/7dacf2ef7dd947d2ad37db118ff60ac5 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/B/7dacf2ef7dd947d2ad37db118ff60ac5 2024-11-17T15:36:51,962 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/B/7dacf2ef7dd947d2ad37db118ff60ac5, entries=150, sequenceid=93, filesize=11.7 K 2024-11-17T15:36:51,964 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/C/3cfef1ced9b3411fad18d6b66b84dca0 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/C/3cfef1ced9b3411fad18d6b66b84dca0 2024-11-17T15:36:51,983 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/C/3cfef1ced9b3411fad18d6b66b84dca0, entries=150, sequenceid=93, filesize=11.7 K 2024-11-17T15:36:51,987 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for a80c4e623aab44bea3cfa33ee4fee8f3 in 1137ms, sequenceid=93, compaction requested=true 2024-11-17T15:36:51,987 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a80c4e623aab44bea3cfa33ee4fee8f3: 2024-11-17T15:36:51,987 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a80c4e623aab44bea3cfa33ee4fee8f3:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T15:36:51,987 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T15:36:51,987 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T15:36:51,987 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a80c4e623aab44bea3cfa33ee4fee8f3:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T15:36:51,987 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T15:36:51,987 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T15:36:51,987 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a80c4e623aab44bea3cfa33ee4fee8f3:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T15:36:51,987 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T15:36:51,990 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T15:36:51,990 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T15:36:51,990 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1541): a80c4e623aab44bea3cfa33ee4fee8f3/B is initiating minor compaction (all files) 2024-11-17T15:36:51,990 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1541): a80c4e623aab44bea3cfa33ee4fee8f3/A is initiating minor compaction (all files) 2024-11-17T15:36:51,990 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a80c4e623aab44bea3cfa33ee4fee8f3/B in TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3. 2024-11-17T15:36:51,990 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a80c4e623aab44bea3cfa33ee4fee8f3/A in TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3. 2024-11-17T15:36:51,991 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/A/bf4940d3105d4ea09ed03e1888157d51, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/A/f4855d7654e548969ed0272bd95df179, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/A/9a30184341e84c8f8a67425ab889451d] into tmpdir=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp, totalSize=35.3 K 2024-11-17T15:36:51,991 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/B/54f3a827533347dc8644ec6bb313dae9, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/B/173babc1b3494a33bd587d631d8d1e14, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/B/7dacf2ef7dd947d2ad37db118ff60ac5] into tmpdir=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp, totalSize=35.3 K 2024-11-17T15:36:51,992 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting bf4940d3105d4ea09ed03e1888157d51, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731857808532 2024-11-17T15:36:51,992 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting 54f3a827533347dc8644ec6bb313dae9, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731857808532 2024-11-17T15:36:51,993 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting f4855d7654e548969ed0272bd95df179, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1731857809770 2024-11-17T15:36:51,993 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting 173babc1b3494a33bd587d631d8d1e14, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1731857809770 2024-11-17T15:36:51,994 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting 7dacf2ef7dd947d2ad37db118ff60ac5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731857810170 2024-11-17T15:36:51,994 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9a30184341e84c8f8a67425ab889451d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731857810170 2024-11-17T15:36:52,019 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a80c4e623aab44bea3cfa33ee4fee8f3#A#compaction#18 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T15:36:52,020 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/A/af2903292537434ab33d6ac72089bda8 is 50, key is test_row_0/A:col10/1731857810846/Put/seqid=0 2024-11-17T15:36:52,036 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a80c4e623aab44bea3cfa33ee4fee8f3#B#compaction#19 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T15:36:52,036 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/B/8e3f17193ab349d99e6256a019e0c395 is 50, key is test_row_0/B:col10/1731857810846/Put/seqid=0 2024-11-17T15:36:52,065 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-17T15:36:52,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3. 2024-11-17T15:36:52,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073741855_1031 (size=12207) 2024-11-17T15:36:52,065 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing a80c4e623aab44bea3cfa33ee4fee8f3 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-17T15:36:52,077 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3. as already flushing 2024-11-17T15:36:52,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(8855): Flush requested on a80c4e623aab44bea3cfa33ee4fee8f3 2024-11-17T15:36:52,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/A/c25a9bb737154ca190744442a7ba939c is 50, key is test_row_0/A:col10/1731857810916/Put/seqid=0 2024-11-17T15:36:52,100 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/A/af2903292537434ab33d6ac72089bda8 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/A/af2903292537434ab33d6ac72089bda8 2024-11-17T15:36:52,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073741856_1032 (size=12207) 2024-11-17T15:36:52,112 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a80c4e623aab44bea3cfa33ee4fee8f3/A of a80c4e623aab44bea3cfa33ee4fee8f3 into af2903292537434ab33d6ac72089bda8(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T15:36:52,113 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a80c4e623aab44bea3cfa33ee4fee8f3: 2024-11-17T15:36:52,113 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., storeName=a80c4e623aab44bea3cfa33ee4fee8f3/A, priority=13, startTime=1731857811987; duration=0sec 2024-11-17T15:36:52,114 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T15:36:52,114 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a80c4e623aab44bea3cfa33ee4fee8f3:A 2024-11-17T15:36:52,114 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T15:36:52,117 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T15:36:52,117 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1541): a80c4e623aab44bea3cfa33ee4fee8f3/C is initiating minor compaction (all files) 2024-11-17T15:36:52,117 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a80c4e623aab44bea3cfa33ee4fee8f3/C in TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3. 2024-11-17T15:36:52,117 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/C/dd28e9fd8ba74381868490f561a28208, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/C/ce9ced83164e42da9aa5e6c22eb49366, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/C/3cfef1ced9b3411fad18d6b66b84dca0] into tmpdir=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp, totalSize=35.3 K 2024-11-17T15:36:52,118 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting dd28e9fd8ba74381868490f561a28208, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731857808532 2024-11-17T15:36:52,119 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting ce9ced83164e42da9aa5e6c22eb49366, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1731857809770 2024-11-17T15:36:52,121 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3cfef1ced9b3411fad18d6b66b84dca0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731857810170 2024-11-17T15:36:52,122 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/B/8e3f17193ab349d99e6256a019e0c395 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/B/8e3f17193ab349d99e6256a019e0c395 2024-11-17T15:36:52,129 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:36:52,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57496 deadline: 1731857872112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 2024-11-17T15:36:52,130 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:52,131 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:52,131 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:36:52,132 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a80c4e623aab44bea3cfa33ee4fee8f3/B of a80c4e623aab44bea3cfa33ee4fee8f3 into 8e3f17193ab349d99e6256a019e0c395(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T15:36:52,132 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a80c4e623aab44bea3cfa33ee4fee8f3: 2024-11-17T15:36:52,132 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., storeName=a80c4e623aab44bea3cfa33ee4fee8f3/B, priority=13, startTime=1731857811987; duration=0sec 2024-11-17T15:36:52,133 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T15:36:52,133 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a80c4e623aab44bea3cfa33ee4fee8f3:B 2024-11-17T15:36:52,135 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:36:52,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57502 deadline: 1731857872122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 2024-11-17T15:36:52,136 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:52,136 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:52,136 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:36:52,135 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:36:52,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57488 deadline: 1731857872116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 2024-11-17T15:36:52,138 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:52,138 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:36:52,138 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:52,138 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:36:52,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57530 deadline: 1731857872126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 2024-11-17T15:36:52,139 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:52,139 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:52,139 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:36:52,139 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:36:52,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57522 deadline: 1731857872130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 2024-11-17T15:36:52,140 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:52,140 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:52,140 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:36:52,142 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a80c4e623aab44bea3cfa33ee4fee8f3#C#compaction#21 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T15:36:52,143 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/C/75e5d5e3cdb34a5c9816734e84436e6d is 50, key is test_row_0/C:col10/1731857810846/Put/seqid=0 2024-11-17T15:36:52,147 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073741857_1033 (size=12001) 2024-11-17T15:36:52,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073741858_1034 (size=12207) 2024-11-17T15:36:52,196 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/C/75e5d5e3cdb34a5c9816734e84436e6d as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/C/75e5d5e3cdb34a5c9816734e84436e6d 2024-11-17T15:36:52,215 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a80c4e623aab44bea3cfa33ee4fee8f3/C of a80c4e623aab44bea3cfa33ee4fee8f3 into 75e5d5e3cdb34a5c9816734e84436e6d(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T15:36:52,215 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a80c4e623aab44bea3cfa33ee4fee8f3: 2024-11-17T15:36:52,215 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., storeName=a80c4e623aab44bea3cfa33ee4fee8f3/C, priority=13, startTime=1731857811987; duration=0sec 2024-11-17T15:36:52,215 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T15:36:52,215 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a80c4e623aab44bea3cfa33ee4fee8f3:C 2024-11-17T15:36:52,241 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:36:52,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57496 deadline: 1731857872233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 2024-11-17T15:36:52,242 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:52,242 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:52,242 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:36:52,246 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:36:52,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57502 deadline: 1731857872243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 2024-11-17T15:36:52,247 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:52,247 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:36:52,247 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:52,247 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:36:52,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57530 deadline: 1731857872244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 2024-11-17T15:36:52,247 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:52,247 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:52,247 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:36:52,247 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:36:52,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57488 deadline: 1731857872244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 2024-11-17T15:36:52,248 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:52,248 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:52,248 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:36:52,251 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:36:52,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57522 deadline: 1731857872244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 2024-11-17T15:36:52,253 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:52,253 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:52,253 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:36:52,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-17T15:36:52,455 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:36:52,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57502 deadline: 1731857872454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 2024-11-17T15:36:52,457 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:36:52,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57530 deadline: 1731857872454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 2024-11-17T15:36:52,458 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:52,458 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:52,458 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:36:52,458 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:52,458 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:36:52,458 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:52,458 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:36:52,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57496 deadline: 1731857872454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 2024-11-17T15:36:52,459 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:52,459 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:52,459 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:36:52,459 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:36:52,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57488 deadline: 1731857872454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 2024-11-17T15:36:52,459 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:52,459 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:52,460 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:36:52,464 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:36:52,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57522 deadline: 1731857872463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 2024-11-17T15:36:52,465 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:52,465 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:52,465 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:36:52,548 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/A/c25a9bb737154ca190744442a7ba939c 2024-11-17T15:36:52,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/B/402182b8978946f2a3953e81848a96d0 is 50, key is test_row_0/B:col10/1731857810916/Put/seqid=0 2024-11-17T15:36:52,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073741859_1035 (size=12001) 2024-11-17T15:36:52,621 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/B/402182b8978946f2a3953e81848a96d0 2024-11-17T15:36:52,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/C/becd22d7319f4fa8ae277234c1dd5f85 is 50, key is test_row_0/C:col10/1731857810916/Put/seqid=0 2024-11-17T15:36:52,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073741860_1036 (size=12001) 2024-11-17T15:36:52,701 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/C/becd22d7319f4fa8ae277234c1dd5f85 2024-11-17T15:36:52,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/A/c25a9bb737154ca190744442a7ba939c as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/A/c25a9bb737154ca190744442a7ba939c 2024-11-17T15:36:52,739 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/A/c25a9bb737154ca190744442a7ba939c, entries=150, sequenceid=114, filesize=11.7 K 2024-11-17T15:36:52,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/B/402182b8978946f2a3953e81848a96d0 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/B/402182b8978946f2a3953e81848a96d0 2024-11-17T15:36:52,759 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/B/402182b8978946f2a3953e81848a96d0, entries=150, sequenceid=114, filesize=11.7 K 2024-11-17T15:36:52,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/C/becd22d7319f4fa8ae277234c1dd5f85 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/C/becd22d7319f4fa8ae277234c1dd5f85 2024-11-17T15:36:52,769 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:36:52,770 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:36:52,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57496 deadline: 1731857872764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 2024-11-17T15:36:52,771 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:36:52,771 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:52,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57530 deadline: 1731857872764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 2024-11-17T15:36:52,771 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:52,771 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:36:52,771 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:52,771 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:36:52,772 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:52,772 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:36:52,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57488 deadline: 1731857872764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 2024-11-17T15:36:52,772 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:52,772 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:52,772 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:36:52,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57502 deadline: 1731857872764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 2024-11-17T15:36:52,773 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:52,773 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:52,774 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:36:52,774 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/C/becd22d7319f4fa8ae277234c1dd5f85, entries=150, sequenceid=114, filesize=11.7 K 2024-11-17T15:36:52,776 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for a80c4e623aab44bea3cfa33ee4fee8f3 in 711ms, sequenceid=114, compaction requested=false 2024-11-17T15:36:52,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for a80c4e623aab44bea3cfa33ee4fee8f3: 2024-11-17T15:36:52,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3. 2024-11-17T15:36:52,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-11-17T15:36:52,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-11-17T15:36:52,785 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-17T15:36:52,785 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5080 sec 2024-11-17T15:36:52,801 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 1.5420 sec 2024-11-17T15:36:52,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(8855): Flush requested on a80c4e623aab44bea3cfa33ee4fee8f3 2024-11-17T15:36:52,814 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T15:36:52,814 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a80c4e623aab44bea3cfa33ee4fee8f3 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-17T15:36:52,827 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/A/c4d24d5dca594a368262ffbebc6cc41b is 50, key is test_row_0/A:col10/1731857812783/Put/seqid=0 2024-11-17T15:36:52,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073741861_1037 (size=14441) 2024-11-17T15:36:52,978 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:36:52,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57522 deadline: 1731857872976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 2024-11-17T15:36:52,979 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:52,979 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:52,979 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:36:53,085 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:36:53,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57522 deadline: 1731857873083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 2024-11-17T15:36:53,086 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:53,086 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:53,086 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:36:53,264 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/A/c4d24d5dca594a368262ffbebc6cc41b 2024-11-17T15:36:53,274 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:36:53,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57496 deadline: 1731857873273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 2024-11-17T15:36:53,277 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:53,277 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:53,277 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:36:53,289 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:36:53,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57530 deadline: 1731857873284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 2024-11-17T15:36:53,290 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:53,290 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:36:53,290 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:53,290 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:36:53,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57502 deadline: 1731857873284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 2024-11-17T15:36:53,290 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:53,290 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:53,290 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:36:53,290 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:36:53,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57488 deadline: 1731857873284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 2024-11-17T15:36:53,291 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:53,291 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:53,291 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:36:53,297 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:36:53,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57522 deadline: 1731857873293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 2024-11-17T15:36:53,298 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:53,298 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:53,298 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:36:53,301 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/B/257005ae561f4a7695b8858b5a2dbd6c is 50, key is test_row_0/B:col10/1731857812783/Put/seqid=0 2024-11-17T15:36:53,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073741862_1038 (size=12051) 2024-11-17T15:36:53,324 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/B/257005ae561f4a7695b8858b5a2dbd6c 2024-11-17T15:36:53,347 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/C/91e5651d57f84b14b08453ad2839a6af is 50, key is test_row_0/C:col10/1731857812783/Put/seqid=0 2024-11-17T15:36:53,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073741863_1039 (size=12051) 2024-11-17T15:36:53,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-17T15:36:53,404 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T15:36:53,405 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T15:36:53,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-11-17T15:36:53,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-17T15:36:53,412 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T15:36:53,417 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T15:36:53,417 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T15:36:53,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-17T15:36:53,571 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-17T15:36:53,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3. 2024-11-17T15:36:53,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3. as already flushing 2024-11-17T15:36:53,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3. 2024-11-17T15:36:53,572 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => a80c4e623aab44bea3cfa33ee4fee8f3, NAME => 'TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:36:53,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => a80c4e623aab44bea3cfa33ee4fee8f3, NAME => 'TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:36:53,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a80c4e623aab44bea3cfa33ee4fee8f3, NAME => 'TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a80c4e623aab44bea3cfa33ee4fee8f3, NAME => 'TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:36:53,605 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:36:53,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57522 deadline: 1731857873603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 2024-11-17T15:36:53,606 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:53,606 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:53,606 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:36:53,727 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-17T15:36:53,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3. 2024-11-17T15:36:53,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3. as already flushing 2024-11-17T15:36:53,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3. 2024-11-17T15:36:53,727 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => a80c4e623aab44bea3cfa33ee4fee8f3, NAME => 'TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:36:53,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => a80c4e623aab44bea3cfa33ee4fee8f3, NAME => 'TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:36:53,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a80c4e623aab44bea3cfa33ee4fee8f3, NAME => 'TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a80c4e623aab44bea3cfa33ee4fee8f3, NAME => 'TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:36:53,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-17T15:36:53,761 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/C/91e5651d57f84b14b08453ad2839a6af 2024-11-17T15:36:53,773 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/A/c4d24d5dca594a368262ffbebc6cc41b as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/A/c4d24d5dca594a368262ffbebc6cc41b 2024-11-17T15:36:53,786 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/A/c4d24d5dca594a368262ffbebc6cc41b, entries=200, sequenceid=133, filesize=14.1 K 2024-11-17T15:36:53,789 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/B/257005ae561f4a7695b8858b5a2dbd6c as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/B/257005ae561f4a7695b8858b5a2dbd6c 2024-11-17T15:36:53,799 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/B/257005ae561f4a7695b8858b5a2dbd6c, entries=150, sequenceid=133, filesize=11.8 K 2024-11-17T15:36:53,800 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/C/91e5651d57f84b14b08453ad2839a6af as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/C/91e5651d57f84b14b08453ad2839a6af 2024-11-17T15:36:53,814 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/C/91e5651d57f84b14b08453ad2839a6af, entries=150, sequenceid=133, filesize=11.8 K 2024-11-17T15:36:53,815 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for a80c4e623aab44bea3cfa33ee4fee8f3 in 1001ms, sequenceid=133, compaction requested=true 2024-11-17T15:36:53,815 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a80c4e623aab44bea3cfa33ee4fee8f3: 2024-11-17T15:36:53,816 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a80c4e623aab44bea3cfa33ee4fee8f3:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T15:36:53,816 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T15:36:53,816 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T15:36:53,816 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T15:36:53,816 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a80c4e623aab44bea3cfa33ee4fee8f3:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T15:36:53,817 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T15:36:53,817 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a80c4e623aab44bea3cfa33ee4fee8f3:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T15:36:53,817 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T15:36:53,818 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38649 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T15:36:53,818 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1541): a80c4e623aab44bea3cfa33ee4fee8f3/A is initiating minor compaction (all files) 2024-11-17T15:36:53,818 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a80c4e623aab44bea3cfa33ee4fee8f3/A in TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3. 2024-11-17T15:36:53,818 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/A/af2903292537434ab33d6ac72089bda8, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/A/c25a9bb737154ca190744442a7ba939c, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/A/c4d24d5dca594a368262ffbebc6cc41b] into tmpdir=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp, totalSize=37.7 K 2024-11-17T15:36:53,819 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting af2903292537434ab33d6ac72089bda8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731857810170 2024-11-17T15:36:53,820 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting c25a9bb737154ca190744442a7ba939c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1731857810903 2024-11-17T15:36:53,820 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36259 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T15:36:53,820 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1541): a80c4e623aab44bea3cfa33ee4fee8f3/B is initiating minor compaction (all files) 2024-11-17T15:36:53,820 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting c4d24d5dca594a368262ffbebc6cc41b, keycount=200, bloomtype=ROW, size=14.1 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731857812118 2024-11-17T15:36:53,820 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a80c4e623aab44bea3cfa33ee4fee8f3/B in TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3. 2024-11-17T15:36:53,821 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/B/8e3f17193ab349d99e6256a019e0c395, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/B/402182b8978946f2a3953e81848a96d0, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/B/257005ae561f4a7695b8858b5a2dbd6c] into tmpdir=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp, totalSize=35.4 K 2024-11-17T15:36:53,822 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting 8e3f17193ab349d99e6256a019e0c395, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731857810170 2024-11-17T15:36:53,823 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting 402182b8978946f2a3953e81848a96d0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1731857810903 2024-11-17T15:36:53,824 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting 257005ae561f4a7695b8858b5a2dbd6c, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731857812118 2024-11-17T15:36:53,843 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a80c4e623aab44bea3cfa33ee4fee8f3#A#compaction#27 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T15:36:53,844 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/A/9371bd1061eb4adda17613446d996e31 is 50, key is test_row_0/A:col10/1731857812783/Put/seqid=0 2024-11-17T15:36:53,854 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a80c4e623aab44bea3cfa33ee4fee8f3#B#compaction#28 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T15:36:53,855 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/B/cc23f2b2fb9d4a5d813bf9cf1db8de13 is 50, key is test_row_0/B:col10/1731857812783/Put/seqid=0 2024-11-17T15:36:53,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073741864_1040 (size=12359) 2024-11-17T15:36:53,882 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-17T15:36:53,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3. 2024-11-17T15:36:53,883 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing a80c4e623aab44bea3cfa33ee4fee8f3 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-17T15:36:53,888 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/A/9371bd1061eb4adda17613446d996e31 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/A/9371bd1061eb4adda17613446d996e31 2024-11-17T15:36:53,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073741865_1041 (size=12359) 2024-11-17T15:36:53,899 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a80c4e623aab44bea3cfa33ee4fee8f3/A of a80c4e623aab44bea3cfa33ee4fee8f3 into 9371bd1061eb4adda17613446d996e31(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T15:36:53,899 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a80c4e623aab44bea3cfa33ee4fee8f3: 2024-11-17T15:36:53,899 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., storeName=a80c4e623aab44bea3cfa33ee4fee8f3/A, priority=13, startTime=1731857813815; duration=0sec 2024-11-17T15:36:53,900 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T15:36:53,900 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a80c4e623aab44bea3cfa33ee4fee8f3:A 2024-11-17T15:36:53,900 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T15:36:53,902 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36259 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T15:36:53,902 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1541): a80c4e623aab44bea3cfa33ee4fee8f3/C is initiating minor compaction (all files) 2024-11-17T15:36:53,902 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a80c4e623aab44bea3cfa33ee4fee8f3/C in TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3. 2024-11-17T15:36:53,902 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/C/75e5d5e3cdb34a5c9816734e84436e6d, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/C/becd22d7319f4fa8ae277234c1dd5f85, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/C/91e5651d57f84b14b08453ad2839a6af] into tmpdir=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp, totalSize=35.4 K 2024-11-17T15:36:53,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/A/f1d805562bc24eacae22cff65e8b3818 is 50, key is test_row_0/A:col10/1731857812969/Put/seqid=0 2024-11-17T15:36:53,903 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 75e5d5e3cdb34a5c9816734e84436e6d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731857810170 2024-11-17T15:36:53,903 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting becd22d7319f4fa8ae277234c1dd5f85, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1731857810903 2024-11-17T15:36:53,904 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 91e5651d57f84b14b08453ad2839a6af, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731857812118 2024-11-17T15:36:53,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073741866_1042 (size=12151) 2024-11-17T15:36:53,921 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a80c4e623aab44bea3cfa33ee4fee8f3#C#compaction#30 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T15:36:53,921 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/C/928d5b7cacba40ad87e24d2af95d5820 is 50, key is test_row_0/C:col10/1731857812783/Put/seqid=0 2024-11-17T15:36:53,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073741867_1043 (size=12359) 2024-11-17T15:36:53,944 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/C/928d5b7cacba40ad87e24d2af95d5820 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/C/928d5b7cacba40ad87e24d2af95d5820 2024-11-17T15:36:53,958 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a80c4e623aab44bea3cfa33ee4fee8f3/C of a80c4e623aab44bea3cfa33ee4fee8f3 into 928d5b7cacba40ad87e24d2af95d5820(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T15:36:53,958 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a80c4e623aab44bea3cfa33ee4fee8f3: 2024-11-17T15:36:53,958 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., storeName=a80c4e623aab44bea3cfa33ee4fee8f3/C, priority=13, startTime=1731857813817; duration=0sec 2024-11-17T15:36:53,958 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T15:36:53,958 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a80c4e623aab44bea3cfa33ee4fee8f3:C 2024-11-17T15:36:54,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-17T15:36:54,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(8855): Flush requested on a80c4e623aab44bea3cfa33ee4fee8f3 2024-11-17T15:36:54,116 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3. as already flushing 2024-11-17T15:36:54,170 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:36:54,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57522 deadline: 1731857874170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 2024-11-17T15:36:54,171 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:54,171 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:54,171 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:36:54,274 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:36:54,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57522 deadline: 1731857874274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 2024-11-17T15:36:54,275 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:54,275 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:54,276 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:36:54,284 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:36:54,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57496 deadline: 1731857874283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 2024-11-17T15:36:54,288 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 7a780d55532c,32955,1731857800648 for 'test_row_0' in a80c4e623aab44bea3cfa33ee4fee8f3 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2172 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T15:36:54,289 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:54,290 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:54,290 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:36:54,308 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/B/cc23f2b2fb9d4a5d813bf9cf1db8de13 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/B/cc23f2b2fb9d4a5d813bf9cf1db8de13 2024-11-17T15:36:54,309 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:36:54,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57530 deadline: 1731857874304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 2024-11-17T15:36:54,309 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:36:54,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57502 deadline: 1731857874304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 2024-11-17T15:36:54,310 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:36:54,309 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 7a780d55532c,32955,1731857800648 for 'test_row_1' in a80c4e623aab44bea3cfa33ee4fee8f3 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2183 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T15:36:54,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57488 deadline: 1731857874304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 2024-11-17T15:36:54,310 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:54,310 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:54,310 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:36:54,311 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 7a780d55532c,32955,1731857800648 for 'test_row_0' in a80c4e623aab44bea3cfa33ee4fee8f3 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2194 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T15:36:54,311 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 7a780d55532c,32955,1731857800648 for 'test_row_0' in a80c4e623aab44bea3cfa33ee4fee8f3 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2189 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T15:36:54,311 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:54,311 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:54,312 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:54,312 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:54,312 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:36:54,312 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:36:54,316 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/A/f1d805562bc24eacae22cff65e8b3818 2024-11-17T15:36:54,331 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a80c4e623aab44bea3cfa33ee4fee8f3/B of a80c4e623aab44bea3cfa33ee4fee8f3 into cc23f2b2fb9d4a5d813bf9cf1db8de13(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T15:36:54,331 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a80c4e623aab44bea3cfa33ee4fee8f3: 2024-11-17T15:36:54,331 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., storeName=a80c4e623aab44bea3cfa33ee4fee8f3/B, priority=13, startTime=1731857813816; duration=0sec 2024-11-17T15:36:54,332 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T15:36:54,332 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a80c4e623aab44bea3cfa33ee4fee8f3:B 2024-11-17T15:36:54,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/B/17d92fa5e77244afa19591d96578ff9a is 50, key is test_row_0/B:col10/1731857812969/Put/seqid=0 2024-11-17T15:36:54,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073741868_1044 (size=12151) 2024-11-17T15:36:54,484 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:36:54,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57522 deadline: 1731857874483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 2024-11-17T15:36:54,485 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:54,485 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:54,485 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:36:54,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-17T15:36:54,752 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/B/17d92fa5e77244afa19591d96578ff9a 2024-11-17T15:36:54,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/C/15a7ee9bbc454ea0ab035b404183d8d0 is 50, key is test_row_0/C:col10/1731857812969/Put/seqid=0 2024-11-17T15:36:54,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073741869_1045 (size=12151) 2024-11-17T15:36:54,802 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:36:54,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57522 deadline: 1731857874796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 2024-11-17T15:36:54,803 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:54,804 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:54,804 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:36:55,198 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/C/15a7ee9bbc454ea0ab035b404183d8d0 2024-11-17T15:36:55,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/A/f1d805562bc24eacae22cff65e8b3818 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/A/f1d805562bc24eacae22cff65e8b3818 2024-11-17T15:36:55,241 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/A/f1d805562bc24eacae22cff65e8b3818, entries=150, sequenceid=153, filesize=11.9 K 2024-11-17T15:36:55,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/B/17d92fa5e77244afa19591d96578ff9a as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/B/17d92fa5e77244afa19591d96578ff9a 2024-11-17T15:36:55,250 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/B/17d92fa5e77244afa19591d96578ff9a, entries=150, sequenceid=153, filesize=11.9 K 2024-11-17T15:36:55,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/C/15a7ee9bbc454ea0ab035b404183d8d0 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/C/15a7ee9bbc454ea0ab035b404183d8d0 2024-11-17T15:36:55,261 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/C/15a7ee9bbc454ea0ab035b404183d8d0, entries=150, sequenceid=153, filesize=11.9 K 2024-11-17T15:36:55,263 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for a80c4e623aab44bea3cfa33ee4fee8f3 in 1379ms, sequenceid=153, compaction requested=false 2024-11-17T15:36:55,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for a80c4e623aab44bea3cfa33ee4fee8f3: 2024-11-17T15:36:55,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3. 2024-11-17T15:36:55,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-11-17T15:36:55,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-11-17T15:36:55,271 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-17T15:36:55,271 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8500 sec 2024-11-17T15:36:55,276 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 1.8690 sec 2024-11-17T15:36:55,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(8855): Flush requested on a80c4e623aab44bea3cfa33ee4fee8f3 2024-11-17T15:36:55,318 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T15:36:55,318 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a80c4e623aab44bea3cfa33ee4fee8f3 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-17T15:36:55,327 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/A/60c5a43971a24642977b31c98205a6d4 is 50, key is test_row_0/A:col10/1731857814168/Put/seqid=0 2024-11-17T15:36:55,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073741870_1046 (size=14541) 2024-11-17T15:36:55,441 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:36:55,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57522 deadline: 1731857875436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 2024-11-17T15:36:55,442 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:55,442 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:55,442 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:36:55,554 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:36:55,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57522 deadline: 1731857875553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 2024-11-17T15:36:55,555 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:55,555 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:55,555 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:36:55,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-17T15:36:55,563 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T15:36:55,564 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T15:36:55,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-11-17T15:36:55,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-17T15:36:55,567 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T15:36:55,568 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T15:36:55,569 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T15:36:55,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-17T15:36:55,723 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-17T15:36:55,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3. 2024-11-17T15:36:55,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3. as already flushing 2024-11-17T15:36:55,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3. 2024-11-17T15:36:55,724 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => a80c4e623aab44bea3cfa33ee4fee8f3, NAME => 'TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:36:55,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => a80c4e623aab44bea3cfa33ee4fee8f3, NAME => 'TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:36:55,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a80c4e623aab44bea3cfa33ee4fee8f3, NAME => 'TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a80c4e623aab44bea3cfa33ee4fee8f3, NAME => 'TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:36:55,750 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/A/60c5a43971a24642977b31c98205a6d4 2024-11-17T15:36:55,768 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:36:55,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57522 deadline: 1731857875763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 2024-11-17T15:36:55,769 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:55,770 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:55,770 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:36:55,770 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/B/edce9a0729d7455dbc263cf9297faef0 is 50, key is test_row_0/B:col10/1731857814168/Put/seqid=0 2024-11-17T15:36:55,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073741871_1047 (size=12151) 2024-11-17T15:36:55,812 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/B/edce9a0729d7455dbc263cf9297faef0 2024-11-17T15:36:55,844 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/C/f45426d1a7ca450795c4b3024a9081ea is 50, key is test_row_0/C:col10/1731857814168/Put/seqid=0 2024-11-17T15:36:55,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-17T15:36:55,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073741872_1048 (size=12151) 2024-11-17T15:36:55,889 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-17T15:36:55,889 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/C/f45426d1a7ca450795c4b3024a9081ea 2024-11-17T15:36:55,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3. 2024-11-17T15:36:55,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3. as already flushing 2024-11-17T15:36:55,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3. 2024-11-17T15:36:55,890 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => a80c4e623aab44bea3cfa33ee4fee8f3, NAME => 'TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:36:55,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => a80c4e623aab44bea3cfa33ee4fee8f3, NAME => 'TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:36:55,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a80c4e623aab44bea3cfa33ee4fee8f3, NAME => 'TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a80c4e623aab44bea3cfa33ee4fee8f3, NAME => 'TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:36:55,899 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/A/60c5a43971a24642977b31c98205a6d4 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/A/60c5a43971a24642977b31c98205a6d4 2024-11-17T15:36:55,916 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/A/60c5a43971a24642977b31c98205a6d4, entries=200, sequenceid=173, filesize=14.2 K 2024-11-17T15:36:55,918 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/B/edce9a0729d7455dbc263cf9297faef0 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/B/edce9a0729d7455dbc263cf9297faef0 2024-11-17T15:36:55,940 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/B/edce9a0729d7455dbc263cf9297faef0, entries=150, sequenceid=173, filesize=11.9 K 2024-11-17T15:36:55,942 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/C/f45426d1a7ca450795c4b3024a9081ea as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/C/f45426d1a7ca450795c4b3024a9081ea 2024-11-17T15:36:55,976 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/C/f45426d1a7ca450795c4b3024a9081ea, entries=150, sequenceid=173, filesize=11.9 K 2024-11-17T15:36:55,980 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for a80c4e623aab44bea3cfa33ee4fee8f3 in 662ms, sequenceid=173, compaction requested=true 2024-11-17T15:36:55,981 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a80c4e623aab44bea3cfa33ee4fee8f3: 2024-11-17T15:36:55,983 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a80c4e623aab44bea3cfa33ee4fee8f3:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T15:36:55,983 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T15:36:55,983 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T15:36:55,983 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T15:36:55,986 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39051 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T15:36:55,986 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1541): a80c4e623aab44bea3cfa33ee4fee8f3/A is initiating minor compaction (all files) 2024-11-17T15:36:55,986 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a80c4e623aab44bea3cfa33ee4fee8f3/A in TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3. 2024-11-17T15:36:55,986 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/A/9371bd1061eb4adda17613446d996e31, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/A/f1d805562bc24eacae22cff65e8b3818, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/A/60c5a43971a24642977b31c98205a6d4] into tmpdir=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp, totalSize=38.1 K 2024-11-17T15:36:55,987 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36661 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T15:36:55,987 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1541): a80c4e623aab44bea3cfa33ee4fee8f3/B is initiating minor compaction (all files) 2024-11-17T15:36:55,987 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a80c4e623aab44bea3cfa33ee4fee8f3/B in TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3. 2024-11-17T15:36:55,987 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/B/cc23f2b2fb9d4a5d813bf9cf1db8de13, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/B/17d92fa5e77244afa19591d96578ff9a, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/B/edce9a0729d7455dbc263cf9297faef0] into tmpdir=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp, totalSize=35.8 K 2024-11-17T15:36:55,988 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting cc23f2b2fb9d4a5d813bf9cf1db8de13, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731857812118 2024-11-17T15:36:55,988 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9371bd1061eb4adda17613446d996e31, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731857812118 2024-11-17T15:36:55,989 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting f1d805562bc24eacae22cff65e8b3818, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1731857812935 2024-11-17T15:36:55,989 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting 17d92fa5e77244afa19591d96578ff9a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1731857812935 2024-11-17T15:36:55,990 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 60c5a43971a24642977b31c98205a6d4, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1731857814136 2024-11-17T15:36:55,990 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting edce9a0729d7455dbc263cf9297faef0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1731857814152 2024-11-17T15:36:56,012 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a80c4e623aab44bea3cfa33ee4fee8f3:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T15:36:56,013 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T15:36:56,013 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a80c4e623aab44bea3cfa33ee4fee8f3:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T15:36:56,013 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T15:36:56,022 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a80c4e623aab44bea3cfa33ee4fee8f3#A#compaction#36 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T15:36:56,023 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/A/1dc6533af38b4cbe847ca261b396f703 is 50, key is test_row_0/A:col10/1731857814168/Put/seqid=0 2024-11-17T15:36:56,039 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a80c4e623aab44bea3cfa33ee4fee8f3#B#compaction#37 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T15:36:56,040 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/B/dcc35d4ce2034fba995cddef4211e265 is 50, key is test_row_0/B:col10/1731857814168/Put/seqid=0 2024-11-17T15:36:56,045 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-17T15:36:56,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3. 2024-11-17T15:36:56,046 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing a80c4e623aab44bea3cfa33ee4fee8f3 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-17T15:36:56,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/A/13ea5809700740d9b5aca2b674dcc378 is 50, key is test_row_0/A:col10/1731857815428/Put/seqid=0 2024-11-17T15:36:56,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(8855): Flush requested on a80c4e623aab44bea3cfa33ee4fee8f3 2024-11-17T15:36:56,085 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3. as already flushing 2024-11-17T15:36:56,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073741874_1050 (size=12561) 2024-11-17T15:36:56,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073741875_1051 (size=12151) 2024-11-17T15:36:56,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073741873_1049 (size=12561) 2024-11-17T15:36:56,183 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/A/1dc6533af38b4cbe847ca261b396f703 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/A/1dc6533af38b4cbe847ca261b396f703 2024-11-17T15:36:56,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-17T15:36:56,209 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a80c4e623aab44bea3cfa33ee4fee8f3/A of a80c4e623aab44bea3cfa33ee4fee8f3 into 1dc6533af38b4cbe847ca261b396f703(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T15:36:56,209 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a80c4e623aab44bea3cfa33ee4fee8f3: 2024-11-17T15:36:56,209 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., storeName=a80c4e623aab44bea3cfa33ee4fee8f3/A, priority=13, startTime=1731857815983; duration=0sec 2024-11-17T15:36:56,209 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T15:36:56,213 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a80c4e623aab44bea3cfa33ee4fee8f3:A 2024-11-17T15:36:56,213 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T15:36:56,215 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36661 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T15:36:56,215 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1541): a80c4e623aab44bea3cfa33ee4fee8f3/C is initiating minor compaction (all files) 2024-11-17T15:36:56,215 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a80c4e623aab44bea3cfa33ee4fee8f3/C in TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3. 2024-11-17T15:36:56,215 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/C/928d5b7cacba40ad87e24d2af95d5820, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/C/15a7ee9bbc454ea0ab035b404183d8d0, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/C/f45426d1a7ca450795c4b3024a9081ea] into tmpdir=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp, totalSize=35.8 K 2024-11-17T15:36:56,216 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 928d5b7cacba40ad87e24d2af95d5820, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731857812118 2024-11-17T15:36:56,217 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 15a7ee9bbc454ea0ab035b404183d8d0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1731857812935 2024-11-17T15:36:56,217 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting f45426d1a7ca450795c4b3024a9081ea, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1731857814152 2024-11-17T15:36:56,255 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a80c4e623aab44bea3cfa33ee4fee8f3#C#compaction#39 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T15:36:56,256 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/C/70f8dadc523b412eba8acb3b47a2bb3c is 50, key is test_row_0/C:col10/1731857814168/Put/seqid=0 2024-11-17T15:36:56,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073741876_1052 (size=12561) 2024-11-17T15:36:56,309 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:36:56,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57496 deadline: 1731857876304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 2024-11-17T15:36:56,310 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 7a780d55532c,32955,1731857800648 for 'test_row_0' in a80c4e623aab44bea3cfa33ee4fee8f3 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4198 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T15:36:56,311 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:56,311 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:56,311 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:36:56,318 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:36:56,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57522 deadline: 1731857876305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 2024-11-17T15:36:56,318 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:36:56,319 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:56,319 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:56,319 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:36:56,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57530 deadline: 1731857876313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 2024-11-17T15:36:56,320 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 7a780d55532c,32955,1731857800648 for 'test_row_1' in a80c4e623aab44bea3cfa33ee4fee8f3 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4194 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T15:36:56,321 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:56,321 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:56,321 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:36:56,327 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:36:56,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57502 deadline: 1731857876323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 2024-11-17T15:36:56,328 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 7a780d55532c,32955,1731857800648 for 'test_row_0' in a80c4e623aab44bea3cfa33ee4fee8f3 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4207 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T15:36:56,329 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:56,329 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:56,329 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:36:56,335 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:36:56,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57488 deadline: 1731857876334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 2024-11-17T15:36:56,336 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 7a780d55532c,32955,1731857800648 for 'test_row_0' in a80c4e623aab44bea3cfa33ee4fee8f3 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4220 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T15:36:56,337 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:56,338 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:56,338 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:36:56,427 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:36:56,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57522 deadline: 1731857876424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 2024-11-17T15:36:56,429 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:56,429 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:56,429 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:36:56,526 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/B/dcc35d4ce2034fba995cddef4211e265 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/B/dcc35d4ce2034fba995cddef4211e265 2024-11-17T15:36:56,530 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/A/13ea5809700740d9b5aca2b674dcc378 2024-11-17T15:36:56,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/B/b81c6acf6ced4e1da499caaad6729146 is 50, key is test_row_0/B:col10/1731857815428/Put/seqid=0 2024-11-17T15:36:56,557 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a80c4e623aab44bea3cfa33ee4fee8f3/B of a80c4e623aab44bea3cfa33ee4fee8f3 into dcc35d4ce2034fba995cddef4211e265(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T15:36:56,557 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a80c4e623aab44bea3cfa33ee4fee8f3: 2024-11-17T15:36:56,558 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., storeName=a80c4e623aab44bea3cfa33ee4fee8f3/B, priority=13, startTime=1731857815983; duration=0sec 2024-11-17T15:36:56,558 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T15:36:56,558 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a80c4e623aab44bea3cfa33ee4fee8f3:B 2024-11-17T15:36:56,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073741877_1053 (size=12151) 2024-11-17T15:36:56,616 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/B/b81c6acf6ced4e1da499caaad6729146 2024-11-17T15:36:56,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/C/8f5ffa1d03ce4f98bbe27cb15aaa48d1 is 50, key is test_row_0/C:col10/1731857815428/Put/seqid=0 2024-11-17T15:36:56,645 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:36:56,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57522 deadline: 1731857876635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 2024-11-17T15:36:56,646 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:56,646 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:56,646 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:36:56,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-17T15:36:56,704 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073741878_1054 (size=12151) 2024-11-17T15:36:56,719 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/C/70f8dadc523b412eba8acb3b47a2bb3c as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/C/70f8dadc523b412eba8acb3b47a2bb3c 2024-11-17T15:36:56,731 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a80c4e623aab44bea3cfa33ee4fee8f3/C of a80c4e623aab44bea3cfa33ee4fee8f3 into 70f8dadc523b412eba8acb3b47a2bb3c(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T15:36:56,731 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a80c4e623aab44bea3cfa33ee4fee8f3: 2024-11-17T15:36:56,732 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., storeName=a80c4e623aab44bea3cfa33ee4fee8f3/C, priority=13, startTime=1731857816013; duration=0sec 2024-11-17T15:36:56,732 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T15:36:56,732 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a80c4e623aab44bea3cfa33ee4fee8f3:C 2024-11-17T15:36:56,954 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:36:56,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57522 deadline: 1731857876953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 2024-11-17T15:36:56,955 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:56,955 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:56,956 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:36:57,106 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/C/8f5ffa1d03ce4f98bbe27cb15aaa48d1 2024-11-17T15:36:57,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/A/13ea5809700740d9b5aca2b674dcc378 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/A/13ea5809700740d9b5aca2b674dcc378 2024-11-17T15:36:57,141 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/A/13ea5809700740d9b5aca2b674dcc378, entries=150, sequenceid=192, filesize=11.9 K 2024-11-17T15:36:57,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/B/b81c6acf6ced4e1da499caaad6729146 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/B/b81c6acf6ced4e1da499caaad6729146 2024-11-17T15:36:57,189 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/B/b81c6acf6ced4e1da499caaad6729146, entries=150, sequenceid=192, filesize=11.9 K 2024-11-17T15:36:57,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/C/8f5ffa1d03ce4f98bbe27cb15aaa48d1 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/C/8f5ffa1d03ce4f98bbe27cb15aaa48d1 2024-11-17T15:36:57,218 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/C/8f5ffa1d03ce4f98bbe27cb15aaa48d1, entries=150, sequenceid=192, filesize=11.9 K 2024-11-17T15:36:57,221 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for a80c4e623aab44bea3cfa33ee4fee8f3 in 1174ms, sequenceid=192, compaction requested=false 2024-11-17T15:36:57,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for a80c4e623aab44bea3cfa33ee4fee8f3: 2024-11-17T15:36:57,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3. 2024-11-17T15:36:57,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-11-17T15:36:57,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-11-17T15:36:57,229 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-17T15:36:57,230 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6570 sec 2024-11-17T15:36:57,233 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 1.6660 sec 2024-11-17T15:36:57,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(8855): Flush requested on a80c4e623aab44bea3cfa33ee4fee8f3 2024-11-17T15:36:57,467 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T15:36:57,467 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a80c4e623aab44bea3cfa33ee4fee8f3 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-17T15:36:57,478 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/A/d73535a23a064d8c926a8b5c93600a45 is 50, key is test_row_0/A:col10/1731857816298/Put/seqid=0 2024-11-17T15:36:57,495 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073741879_1055 (size=12151) 2024-11-17T15:36:57,501 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=213 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/A/d73535a23a064d8c926a8b5c93600a45 2024-11-17T15:36:57,539 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/B/ef6a6b683c8546949102bbbb2136347d is 50, key is test_row_0/B:col10/1731857816298/Put/seqid=0 2024-11-17T15:36:57,584 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073741880_1056 (size=12151) 2024-11-17T15:36:57,585 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=213 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/B/ef6a6b683c8546949102bbbb2136347d 2024-11-17T15:36:57,595 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:36:57,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57522 deadline: 1731857877591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 2024-11-17T15:36:57,596 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:57,596 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:57,596 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:36:57,615 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/C/d5914610221d414db8717795a819386c is 50, key is test_row_0/C:col10/1731857816298/Put/seqid=0 2024-11-17T15:36:57,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073741881_1057 (size=12151) 2024-11-17T15:36:57,705 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:36:57,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57522 deadline: 1731857877703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 2024-11-17T15:36:57,706 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:57,706 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:57,706 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:36:57,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-17T15:36:57,725 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T15:36:57,725 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T15:36:57,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-11-17T15:36:57,729 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T15:36:57,730 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T15:36:57,730 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T15:36:57,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-17T15:36:57,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-17T15:36:57,891 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-17T15:36:57,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3. 2024-11-17T15:36:57,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3. as already flushing 2024-11-17T15:36:57,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3. 2024-11-17T15:36:57,892 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => a80c4e623aab44bea3cfa33ee4fee8f3, NAME => 'TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:36:57,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => a80c4e623aab44bea3cfa33ee4fee8f3, NAME => 'TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:36:57,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a80c4e623aab44bea3cfa33ee4fee8f3, NAME => 'TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a80c4e623aab44bea3cfa33ee4fee8f3, NAME => 'TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:36:57,916 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:36:57,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57522 deadline: 1731857877913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 2024-11-17T15:36:57,918 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:57,918 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:57,918 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:36:58,046 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-17T15:36:58,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3. 2024-11-17T15:36:58,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3. as already flushing 2024-11-17T15:36:58,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3. 2024-11-17T15:36:58,046 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => a80c4e623aab44bea3cfa33ee4fee8f3, NAME => 'TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:36:58,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => a80c4e623aab44bea3cfa33ee4fee8f3, NAME => 'TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:36:58,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a80c4e623aab44bea3cfa33ee4fee8f3, NAME => 'TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a80c4e623aab44bea3cfa33ee4fee8f3, NAME => 'TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:36:58,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-17T15:36:58,061 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=213 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/C/d5914610221d414db8717795a819386c 2024-11-17T15:36:58,076 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/A/d73535a23a064d8c926a8b5c93600a45 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/A/d73535a23a064d8c926a8b5c93600a45 2024-11-17T15:36:58,095 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/A/d73535a23a064d8c926a8b5c93600a45, entries=150, sequenceid=213, filesize=11.9 K 2024-11-17T15:36:58,097 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/B/ef6a6b683c8546949102bbbb2136347d as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/B/ef6a6b683c8546949102bbbb2136347d 2024-11-17T15:36:58,115 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/B/ef6a6b683c8546949102bbbb2136347d, entries=150, sequenceid=213, filesize=11.9 K 2024-11-17T15:36:58,117 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/C/d5914610221d414db8717795a819386c as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/C/d5914610221d414db8717795a819386c 2024-11-17T15:36:58,124 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/C/d5914610221d414db8717795a819386c, entries=150, sequenceid=213, filesize=11.9 K 2024-11-17T15:36:58,126 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for a80c4e623aab44bea3cfa33ee4fee8f3 in 659ms, sequenceid=213, compaction requested=true 2024-11-17T15:36:58,126 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a80c4e623aab44bea3cfa33ee4fee8f3: 2024-11-17T15:36:58,126 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a80c4e623aab44bea3cfa33ee4fee8f3:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T15:36:58,126 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T15:36:58,126 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a80c4e623aab44bea3cfa33ee4fee8f3:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T15:36:58,126 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T15:36:58,126 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T15:36:58,126 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a80c4e623aab44bea3cfa33ee4fee8f3:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T15:36:58,126 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-17T15:36:58,127 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T15:36:58,135 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T15:36:58,135 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1541): a80c4e623aab44bea3cfa33ee4fee8f3/A is initiating minor compaction (all files) 2024-11-17T15:36:58,135 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a80c4e623aab44bea3cfa33ee4fee8f3/A in TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3. 2024-11-17T15:36:58,136 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/A/1dc6533af38b4cbe847ca261b396f703, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/A/13ea5809700740d9b5aca2b674dcc378, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/A/d73535a23a064d8c926a8b5c93600a45] into tmpdir=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp, totalSize=36.0 K 2024-11-17T15:36:58,136 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T15:36:58,136 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1541): a80c4e623aab44bea3cfa33ee4fee8f3/B is initiating minor compaction (all files) 2024-11-17T15:36:58,136 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a80c4e623aab44bea3cfa33ee4fee8f3/B in TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3. 2024-11-17T15:36:58,137 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/B/dcc35d4ce2034fba995cddef4211e265, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/B/b81c6acf6ced4e1da499caaad6729146, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/B/ef6a6b683c8546949102bbbb2136347d] into tmpdir=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp, totalSize=36.0 K 2024-11-17T15:36:58,137 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting 1dc6533af38b4cbe847ca261b396f703, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1731857814152 2024-11-17T15:36:58,137 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting dcc35d4ce2034fba995cddef4211e265, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1731857814152 2024-11-17T15:36:58,138 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting b81c6acf6ced4e1da499caaad6729146, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1731857815415 2024-11-17T15:36:58,138 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting 13ea5809700740d9b5aca2b674dcc378, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1731857815415 2024-11-17T15:36:58,139 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting ef6a6b683c8546949102bbbb2136347d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1731857816268 2024-11-17T15:36:58,141 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting d73535a23a064d8c926a8b5c93600a45, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1731857816268 2024-11-17T15:36:58,159 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a80c4e623aab44bea3cfa33ee4fee8f3#B#compaction#45 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T15:36:58,160 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/B/a057189da82644cb9ca87b340db3879c is 50, key is test_row_0/B:col10/1731857816298/Put/seqid=0 2024-11-17T15:36:58,161 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a80c4e623aab44bea3cfa33ee4fee8f3#A#compaction#46 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T15:36:58,162 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/A/a910fc9b9ddd45e6b89cee0071ea5f37 is 50, key is test_row_0/A:col10/1731857816298/Put/seqid=0 2024-11-17T15:36:58,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073741882_1058 (size=12663) 2024-11-17T15:36:58,199 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/A/a910fc9b9ddd45e6b89cee0071ea5f37 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/A/a910fc9b9ddd45e6b89cee0071ea5f37 2024-11-17T15:36:58,201 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-17T15:36:58,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3. 2024-11-17T15:36:58,201 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing a80c4e623aab44bea3cfa33ee4fee8f3 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-17T15:36:58,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073741883_1059 (size=12663) 2024-11-17T15:36:58,210 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a80c4e623aab44bea3cfa33ee4fee8f3/A of a80c4e623aab44bea3cfa33ee4fee8f3 into a910fc9b9ddd45e6b89cee0071ea5f37(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T15:36:58,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/A/b682e790f5a84a3e85029088f19c20aa is 50, key is test_row_0/A:col10/1731857817571/Put/seqid=0 2024-11-17T15:36:58,211 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a80c4e623aab44bea3cfa33ee4fee8f3: 2024-11-17T15:36:58,211 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., storeName=a80c4e623aab44bea3cfa33ee4fee8f3/A, priority=13, startTime=1731857818126; duration=0sec 2024-11-17T15:36:58,211 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T15:36:58,211 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a80c4e623aab44bea3cfa33ee4fee8f3:A 2024-11-17T15:36:58,211 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T15:36:58,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073741884_1060 (size=12151) 2024-11-17T15:36:58,219 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/B/a057189da82644cb9ca87b340db3879c as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/B/a057189da82644cb9ca87b340db3879c 2024-11-17T15:36:58,221 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T15:36:58,221 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1541): a80c4e623aab44bea3cfa33ee4fee8f3/C is initiating minor compaction (all files) 2024-11-17T15:36:58,221 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a80c4e623aab44bea3cfa33ee4fee8f3/C in TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3. 2024-11-17T15:36:58,221 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/C/70f8dadc523b412eba8acb3b47a2bb3c, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/C/8f5ffa1d03ce4f98bbe27cb15aaa48d1, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/C/d5914610221d414db8717795a819386c] into tmpdir=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp, totalSize=36.0 K 2024-11-17T15:36:58,222 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting 70f8dadc523b412eba8acb3b47a2bb3c, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1731857814152 2024-11-17T15:36:58,223 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting 8f5ffa1d03ce4f98bbe27cb15aaa48d1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1731857815415 2024-11-17T15:36:58,223 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting d5914610221d414db8717795a819386c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1731857816268 2024-11-17T15:36:58,227 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=231 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/A/b682e790f5a84a3e85029088f19c20aa 2024-11-17T15:36:58,234 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a80c4e623aab44bea3cfa33ee4fee8f3/B of a80c4e623aab44bea3cfa33ee4fee8f3 into a057189da82644cb9ca87b340db3879c(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T15:36:58,234 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a80c4e623aab44bea3cfa33ee4fee8f3: 2024-11-17T15:36:58,234 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., storeName=a80c4e623aab44bea3cfa33ee4fee8f3/B, priority=13, startTime=1731857818126; duration=0sec 2024-11-17T15:36:58,235 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T15:36:58,235 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a80c4e623aab44bea3cfa33ee4fee8f3:B 2024-11-17T15:36:58,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(8855): Flush requested on a80c4e623aab44bea3cfa33ee4fee8f3 2024-11-17T15:36:58,235 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3. as already flushing 2024-11-17T15:36:58,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/B/ffdd0480018b4784b14c6bc32708809f is 50, key is test_row_0/B:col10/1731857817571/Put/seqid=0 2024-11-17T15:36:58,254 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a80c4e623aab44bea3cfa33ee4fee8f3#C#compaction#49 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T15:36:58,255 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/C/f2883ee022f14d4aaffca2deacca3e05 is 50, key is test_row_0/C:col10/1731857816298/Put/seqid=0 2024-11-17T15:36:58,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073741885_1061 (size=12151) 2024-11-17T15:36:58,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073741886_1062 (size=12663) 2024-11-17T15:36:58,339 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:36:58,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57522 deadline: 1731857878334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 2024-11-17T15:36:58,341 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:58,341 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:58,341 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:36:58,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-17T15:36:58,446 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:36:58,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57522 deadline: 1731857878443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 2024-11-17T15:36:58,447 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:58,447 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:58,447 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:36:58,654 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:36:58,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57522 deadline: 1731857878653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 2024-11-17T15:36:58,655 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:58,655 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:58,655 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:36:58,693 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=231 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/B/ffdd0480018b4784b14c6bc32708809f 2024-11-17T15:36:58,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/C/fc675b3661dd4d4f9512b303846c4f51 is 50, key is test_row_0/C:col10/1731857817571/Put/seqid=0 2024-11-17T15:36:58,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073741887_1063 (size=12151) 2024-11-17T15:36:58,714 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=231 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/C/fc675b3661dd4d4f9512b303846c4f51 2024-11-17T15:36:58,732 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/C/f2883ee022f14d4aaffca2deacca3e05 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/C/f2883ee022f14d4aaffca2deacca3e05 2024-11-17T15:36:58,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/A/b682e790f5a84a3e85029088f19c20aa as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/A/b682e790f5a84a3e85029088f19c20aa 2024-11-17T15:36:58,743 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/A/b682e790f5a84a3e85029088f19c20aa, entries=150, sequenceid=231, filesize=11.9 K 2024-11-17T15:36:58,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/B/ffdd0480018b4784b14c6bc32708809f as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/B/ffdd0480018b4784b14c6bc32708809f 2024-11-17T15:36:58,755 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a80c4e623aab44bea3cfa33ee4fee8f3/C of a80c4e623aab44bea3cfa33ee4fee8f3 into f2883ee022f14d4aaffca2deacca3e05(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T15:36:58,755 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a80c4e623aab44bea3cfa33ee4fee8f3: 2024-11-17T15:36:58,755 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., storeName=a80c4e623aab44bea3cfa33ee4fee8f3/C, priority=13, startTime=1731857818126; duration=0sec 2024-11-17T15:36:58,756 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T15:36:58,756 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a80c4e623aab44bea3cfa33ee4fee8f3:C 2024-11-17T15:36:58,757 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/B/ffdd0480018b4784b14c6bc32708809f, entries=150, sequenceid=231, filesize=11.9 K 2024-11-17T15:36:58,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/C/fc675b3661dd4d4f9512b303846c4f51 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/C/fc675b3661dd4d4f9512b303846c4f51 2024-11-17T15:36:58,769 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/C/fc675b3661dd4d4f9512b303846c4f51, entries=150, sequenceid=231, filesize=11.9 K 2024-11-17T15:36:58,772 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for a80c4e623aab44bea3cfa33ee4fee8f3 in 571ms, sequenceid=231, compaction requested=false 2024-11-17T15:36:58,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for a80c4e623aab44bea3cfa33ee4fee8f3: 2024-11-17T15:36:58,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3. 2024-11-17T15:36:58,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-11-17T15:36:58,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-11-17T15:36:58,781 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-11-17T15:36:58,781 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0470 sec 2024-11-17T15:36:58,791 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 1.0590 sec 2024-11-17T15:36:58,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-17T15:36:58,881 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T15:36:58,882 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T15:36:58,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-11-17T15:36:58,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-17T15:36:58,896 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T15:36:58,898 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T15:36:58,898 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T15:36:58,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(8855): Flush requested on a80c4e623aab44bea3cfa33ee4fee8f3 2024-11-17T15:36:58,972 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T15:36:58,973 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a80c4e623aab44bea3cfa33ee4fee8f3 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-17T15:36:58,986 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/A/fcbd0df251c94b3ab8d0bba99e7e16c6 is 50, key is test_row_0/A:col10/1731857818967/Put/seqid=0 2024-11-17T15:36:59,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-17T15:36:59,024 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073741888_1064 (size=12151) 2024-11-17T15:36:59,025 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/A/fcbd0df251c94b3ab8d0bba99e7e16c6 2024-11-17T15:36:59,047 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/B/8e27803435684b4d9c7df4ea05b2201d is 50, key is test_row_0/B:col10/1731857818967/Put/seqid=0 2024-11-17T15:36:59,054 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-17T15:36:59,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3. 2024-11-17T15:36:59,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3. as already flushing 2024-11-17T15:36:59,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3. 2024-11-17T15:36:59,054 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => a80c4e623aab44bea3cfa33ee4fee8f3, NAME => 'TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:36:59,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => a80c4e623aab44bea3cfa33ee4fee8f3, NAME => 'TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:36:59,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a80c4e623aab44bea3cfa33ee4fee8f3, NAME => 'TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a80c4e623aab44bea3cfa33ee4fee8f3, NAME => 'TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:36:59,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073741889_1065 (size=12151) 2024-11-17T15:36:59,080 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/B/8e27803435684b4d9c7df4ea05b2201d 2024-11-17T15:36:59,094 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/C/2b647c6640ac4dc0a80050c7dfc5274d is 50, key is test_row_0/C:col10/1731857818967/Put/seqid=0 2024-11-17T15:36:59,108 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:36:59,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57522 deadline: 1731857879105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 2024-11-17T15:36:59,112 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:59,112 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:59,112 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:36:59,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073741890_1066 (size=12151) 2024-11-17T15:36:59,210 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-17T15:36:59,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3. 2024-11-17T15:36:59,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3. as already flushing 2024-11-17T15:36:59,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3. 2024-11-17T15:36:59,211 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => a80c4e623aab44bea3cfa33ee4fee8f3, NAME => 'TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:36:59,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => a80c4e623aab44bea3cfa33ee4fee8f3, NAME => 'TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:36:59,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a80c4e623aab44bea3cfa33ee4fee8f3, NAME => 'TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a80c4e623aab44bea3cfa33ee4fee8f3, NAME => 'TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:36:59,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-17T15:36:59,246 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:36:59,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57522 deadline: 1731857879244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 2024-11-17T15:36:59,247 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:59,247 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:59,247 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:36:59,365 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-17T15:36:59,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3. 2024-11-17T15:36:59,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3. as already flushing 2024-11-17T15:36:59,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3. 2024-11-17T15:36:59,365 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => a80c4e623aab44bea3cfa33ee4fee8f3, NAME => 'TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:36:59,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => a80c4e623aab44bea3cfa33ee4fee8f3, NAME => 'TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:36:59,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a80c4e623aab44bea3cfa33ee4fee8f3, NAME => 'TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a80c4e623aab44bea3cfa33ee4fee8f3, NAME => 'TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:36:59,453 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:36:59,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57522 deadline: 1731857879453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 2024-11-17T15:36:59,454 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:59,454 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:59,454 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:36:59,516 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/C/2b647c6640ac4dc0a80050c7dfc5274d 2024-11-17T15:36:59,519 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-17T15:36:59,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3. 2024-11-17T15:36:59,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3. as already flushing 2024-11-17T15:36:59,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3. 2024-11-17T15:36:59,524 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => a80c4e623aab44bea3cfa33ee4fee8f3, NAME => 'TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:36:59,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => a80c4e623aab44bea3cfa33ee4fee8f3, NAME => 'TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:36:59,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a80c4e623aab44bea3cfa33ee4fee8f3, NAME => 'TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a80c4e623aab44bea3cfa33ee4fee8f3, NAME => 'TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:36:59,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-17T15:36:59,535 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/A/fcbd0df251c94b3ab8d0bba99e7e16c6 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/A/fcbd0df251c94b3ab8d0bba99e7e16c6 2024-11-17T15:36:59,544 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/A/fcbd0df251c94b3ab8d0bba99e7e16c6, entries=150, sequenceid=253, filesize=11.9 K 2024-11-17T15:36:59,547 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/B/8e27803435684b4d9c7df4ea05b2201d as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/B/8e27803435684b4d9c7df4ea05b2201d 2024-11-17T15:36:59,555 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/B/8e27803435684b4d9c7df4ea05b2201d, entries=150, sequenceid=253, filesize=11.9 K 2024-11-17T15:36:59,556 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/C/2b647c6640ac4dc0a80050c7dfc5274d as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/C/2b647c6640ac4dc0a80050c7dfc5274d 2024-11-17T15:36:59,562 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/C/2b647c6640ac4dc0a80050c7dfc5274d, entries=150, sequenceid=253, filesize=11.9 K 2024-11-17T15:36:59,564 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for a80c4e623aab44bea3cfa33ee4fee8f3 in 591ms, sequenceid=253, compaction requested=true 2024-11-17T15:36:59,564 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a80c4e623aab44bea3cfa33ee4fee8f3: 2024-11-17T15:36:59,564 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T15:36:59,564 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a80c4e623aab44bea3cfa33ee4fee8f3:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T15:36:59,564 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T15:36:59,564 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T15:36:59,565 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a80c4e623aab44bea3cfa33ee4fee8f3:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T15:36:59,565 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T15:36:59,565 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a80c4e623aab44bea3cfa33ee4fee8f3:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T15:36:59,565 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T15:36:59,566 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T15:36:59,566 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1541): a80c4e623aab44bea3cfa33ee4fee8f3/A is initiating minor compaction (all files) 2024-11-17T15:36:59,566 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a80c4e623aab44bea3cfa33ee4fee8f3/A in TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3. 2024-11-17T15:36:59,566 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/A/a910fc9b9ddd45e6b89cee0071ea5f37, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/A/b682e790f5a84a3e85029088f19c20aa, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/A/fcbd0df251c94b3ab8d0bba99e7e16c6] into tmpdir=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp, totalSize=36.1 K 2024-11-17T15:36:59,566 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T15:36:59,566 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1541): a80c4e623aab44bea3cfa33ee4fee8f3/B is initiating minor compaction (all files) 2024-11-17T15:36:59,566 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a80c4e623aab44bea3cfa33ee4fee8f3/B in TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3. 2024-11-17T15:36:59,567 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/B/a057189da82644cb9ca87b340db3879c, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/B/ffdd0480018b4784b14c6bc32708809f, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/B/8e27803435684b4d9c7df4ea05b2201d] into tmpdir=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp, totalSize=36.1 K 2024-11-17T15:36:59,567 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting a910fc9b9ddd45e6b89cee0071ea5f37, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1731857816268 2024-11-17T15:36:59,567 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting a057189da82644cb9ca87b340db3879c, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1731857816268 2024-11-17T15:36:59,568 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting b682e790f5a84a3e85029088f19c20aa, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1731857817571 2024-11-17T15:36:59,568 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting fcbd0df251c94b3ab8d0bba99e7e16c6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1731857818314 2024-11-17T15:36:59,568 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting ffdd0480018b4784b14c6bc32708809f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1731857817571 2024-11-17T15:36:59,569 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting 8e27803435684b4d9c7df4ea05b2201d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1731857818314 2024-11-17T15:36:59,581 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a80c4e623aab44bea3cfa33ee4fee8f3#A#compaction#54 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T15:36:59,582 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/A/5c7e5227e0aa4248ad3d263081aae7de is 50, key is test_row_0/A:col10/1731857818967/Put/seqid=0 2024-11-17T15:36:59,592 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a80c4e623aab44bea3cfa33ee4fee8f3#B#compaction#55 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T15:36:59,592 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/B/41c765f65c0a4c81bcfce4ca50c92d60 is 50, key is test_row_0/B:col10/1731857818967/Put/seqid=0 2024-11-17T15:36:59,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073741891_1067 (size=12765) 2024-11-17T15:36:59,632 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/A/5c7e5227e0aa4248ad3d263081aae7de as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/A/5c7e5227e0aa4248ad3d263081aae7de 2024-11-17T15:36:59,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073741892_1068 (size=12765) 2024-11-17T15:36:59,642 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/B/41c765f65c0a4c81bcfce4ca50c92d60 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/B/41c765f65c0a4c81bcfce4ca50c92d60 2024-11-17T15:36:59,644 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a80c4e623aab44bea3cfa33ee4fee8f3/A of a80c4e623aab44bea3cfa33ee4fee8f3 into 5c7e5227e0aa4248ad3d263081aae7de(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T15:36:59,644 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a80c4e623aab44bea3cfa33ee4fee8f3: 2024-11-17T15:36:59,645 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., storeName=a80c4e623aab44bea3cfa33ee4fee8f3/A, priority=13, startTime=1731857819564; duration=0sec 2024-11-17T15:36:59,645 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T15:36:59,645 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a80c4e623aab44bea3cfa33ee4fee8f3:A 2024-11-17T15:36:59,645 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T15:36:59,647 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T15:36:59,647 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1541): a80c4e623aab44bea3cfa33ee4fee8f3/C is initiating minor compaction (all files) 2024-11-17T15:36:59,648 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a80c4e623aab44bea3cfa33ee4fee8f3/C in TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3. 2024-11-17T15:36:59,648 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/C/f2883ee022f14d4aaffca2deacca3e05, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/C/fc675b3661dd4d4f9512b303846c4f51, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/C/2b647c6640ac4dc0a80050c7dfc5274d] into tmpdir=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp, totalSize=36.1 K 2024-11-17T15:36:59,652 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting f2883ee022f14d4aaffca2deacca3e05, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1731857816268 2024-11-17T15:36:59,659 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting fc675b3661dd4d4f9512b303846c4f51, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1731857817571 2024-11-17T15:36:59,659 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a80c4e623aab44bea3cfa33ee4fee8f3/B of a80c4e623aab44bea3cfa33ee4fee8f3 into 41c765f65c0a4c81bcfce4ca50c92d60(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T15:36:59,659 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a80c4e623aab44bea3cfa33ee4fee8f3: 2024-11-17T15:36:59,659 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., storeName=a80c4e623aab44bea3cfa33ee4fee8f3/B, priority=13, startTime=1731857819564; duration=0sec 2024-11-17T15:36:59,660 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T15:36:59,660 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a80c4e623aab44bea3cfa33ee4fee8f3:B 2024-11-17T15:36:59,660 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2b647c6640ac4dc0a80050c7dfc5274d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1731857818314 2024-11-17T15:36:59,671 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a80c4e623aab44bea3cfa33ee4fee8f3#C#compaction#56 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T15:36:59,672 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/C/2f80a378a9834320b80026f1275588d3 is 50, key is test_row_0/C:col10/1731857818967/Put/seqid=0 2024-11-17T15:36:59,677 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-17T15:36:59,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3. 2024-11-17T15:36:59,678 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing a80c4e623aab44bea3cfa33ee4fee8f3 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-17T15:36:59,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/A/dd4553c4191c4bbb81a5b45b9236645b is 50, key is test_row_0/A:col10/1731857819100/Put/seqid=0 2024-11-17T15:36:59,702 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073741893_1069 (size=12765) 2024-11-17T15:36:59,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073741894_1070 (size=12301) 2024-11-17T15:36:59,770 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3. as already flushing 2024-11-17T15:36:59,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(8855): Flush requested on a80c4e623aab44bea3cfa33ee4fee8f3 2024-11-17T15:36:59,911 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:36:59,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57522 deadline: 1731857879907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 2024-11-17T15:36:59,912 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:59,912 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:36:59,912 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:37:00,028 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:00,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57522 deadline: 1731857880024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:00,028 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:00,028 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:00,029 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:37:00,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-17T15:37:00,115 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/C/2f80a378a9834320b80026f1275588d3 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/C/2f80a378a9834320b80026f1275588d3 2024-11-17T15:37:00,124 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=272 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/A/dd4553c4191c4bbb81a5b45b9236645b 2024-11-17T15:37:00,124 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a80c4e623aab44bea3cfa33ee4fee8f3/C of a80c4e623aab44bea3cfa33ee4fee8f3 into 2f80a378a9834320b80026f1275588d3(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T15:37:00,124 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a80c4e623aab44bea3cfa33ee4fee8f3: 2024-11-17T15:37:00,124 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., storeName=a80c4e623aab44bea3cfa33ee4fee8f3/C, priority=13, startTime=1731857819565; duration=0sec 2024-11-17T15:37:00,124 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T15:37:00,124 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a80c4e623aab44bea3cfa33ee4fee8f3:C 2024-11-17T15:37:00,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/B/7b9e1176d8214ad8b9453f19e186a331 is 50, key is test_row_0/B:col10/1731857819100/Put/seqid=0 2024-11-17T15:37:00,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073741895_1071 (size=12301) 2024-11-17T15:37:00,235 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:00,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57522 deadline: 1731857880233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:00,235 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:00,236 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:00,236 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:37:00,345 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:00,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57496 deadline: 1731857880343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:00,346 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 7a780d55532c,32955,1731857800648 for 'test_row_0' in a80c4e623aab44bea3cfa33ee4fee8f3 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8234 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T15:37:00,347 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:00,347 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:00,347 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:37:00,360 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:00,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57488 deadline: 1731857880353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:00,362 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 7a780d55532c,32955,1731857800648 for 'test_row_0' in a80c4e623aab44bea3cfa33ee4fee8f3 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8246 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T15:37:00,363 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:00,363 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:00,363 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:37:00,366 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:00,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57502 deadline: 1731857880364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:00,367 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 7a780d55532c,32955,1731857800648 for 'test_row_0' in a80c4e623aab44bea3cfa33ee4fee8f3 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8245 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T15:37:00,367 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:00,367 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:00,367 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:37:00,368 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:00,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57530 deadline: 1731857880364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:00,369 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 7a780d55532c,32955,1731857800648 for 'test_row_1' in a80c4e623aab44bea3cfa33ee4fee8f3 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8243 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T15:37:00,370 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:00,370 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:00,370 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:37:00,544 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:00,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57522 deadline: 1731857880543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:00,545 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:00,545 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:00,545 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:37:00,553 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=272 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/B/7b9e1176d8214ad8b9453f19e186a331 2024-11-17T15:37:00,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/C/59133d0cdf4344bfb2b222c734787150 is 50, key is test_row_0/C:col10/1731857819100/Put/seqid=0 2024-11-17T15:37:00,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073741896_1072 (size=12301) 2024-11-17T15:37:00,595 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=272 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/C/59133d0cdf4344bfb2b222c734787150 2024-11-17T15:37:00,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/A/dd4553c4191c4bbb81a5b45b9236645b as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/A/dd4553c4191c4bbb81a5b45b9236645b 2024-11-17T15:37:00,614 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/A/dd4553c4191c4bbb81a5b45b9236645b, entries=150, sequenceid=272, filesize=12.0 K 2024-11-17T15:37:00,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/B/7b9e1176d8214ad8b9453f19e186a331 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/B/7b9e1176d8214ad8b9453f19e186a331 2024-11-17T15:37:00,628 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/B/7b9e1176d8214ad8b9453f19e186a331, entries=150, sequenceid=272, filesize=12.0 K 2024-11-17T15:37:00,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/C/59133d0cdf4344bfb2b222c734787150 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/C/59133d0cdf4344bfb2b222c734787150 2024-11-17T15:37:00,639 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/C/59133d0cdf4344bfb2b222c734787150, entries=150, sequenceid=272, filesize=12.0 K 2024-11-17T15:37:00,641 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for a80c4e623aab44bea3cfa33ee4fee8f3 in 962ms, sequenceid=272, compaction requested=false 2024-11-17T15:37:00,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for a80c4e623aab44bea3cfa33ee4fee8f3: 2024-11-17T15:37:00,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3. 2024-11-17T15:37:00,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-11-17T15:37:00,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-11-17T15:37:00,646 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-17T15:37:00,647 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7460 sec 2024-11-17T15:37:00,650 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 1.7660 sec 2024-11-17T15:37:01,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-17T15:37:01,056 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T15:37:01,056 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T15:37:01,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-11-17T15:37:01,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-17T15:37:01,060 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T15:37:01,062 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T15:37:01,062 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T15:37:01,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(8855): Flush requested on a80c4e623aab44bea3cfa33ee4fee8f3 2024-11-17T15:37:01,063 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T15:37:01,063 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a80c4e623aab44bea3cfa33ee4fee8f3 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-17T15:37:01,083 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/A/3429524979594aff8b5088c73a4644fb is 50, key is test_row_0/A:col10/1731857819885/Put/seqid=0 2024-11-17T15:37:01,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073741897_1073 (size=12301) 2024-11-17T15:37:01,114 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/A/3429524979594aff8b5088c73a4644fb 2024-11-17T15:37:01,130 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/B/d8f232cdcdfd4c6c88d6f781bb680cb2 is 50, key is test_row_0/B:col10/1731857819885/Put/seqid=0 2024-11-17T15:37:01,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073741898_1074 (size=12301) 2024-11-17T15:37:01,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-17T15:37:01,167 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:01,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 210 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57522 deadline: 1731857881167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:01,168 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:01,168 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:01,168 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:37:01,225 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-17T15:37:01,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3. 2024-11-17T15:37:01,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3. as already flushing 2024-11-17T15:37:01,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3. 2024-11-17T15:37:01,225 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => a80c4e623aab44bea3cfa33ee4fee8f3, NAME => 'TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:37:01,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => a80c4e623aab44bea3cfa33ee4fee8f3, NAME => 'TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:37:01,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a80c4e623aab44bea3cfa33ee4fee8f3, NAME => 'TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a80c4e623aab44bea3cfa33ee4fee8f3, NAME => 'TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:37:01,278 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:01,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 211 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57522 deadline: 1731857881274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:01,278 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:01,278 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:01,279 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:37:01,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-17T15:37:01,382 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-17T15:37:01,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3. 2024-11-17T15:37:01,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3. as already flushing 2024-11-17T15:37:01,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3. 2024-11-17T15:37:01,383 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => a80c4e623aab44bea3cfa33ee4fee8f3, NAME => 'TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:37:01,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => a80c4e623aab44bea3cfa33ee4fee8f3, NAME => 'TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:37:01,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a80c4e623aab44bea3cfa33ee4fee8f3, NAME => 'TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a80c4e623aab44bea3cfa33ee4fee8f3, NAME => 'TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:37:01,486 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:01,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 212 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57522 deadline: 1731857881483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:01,487 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:01,487 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:01,487 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:37:01,544 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-17T15:37:01,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3. 2024-11-17T15:37:01,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3. as already flushing 2024-11-17T15:37:01,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3. 2024-11-17T15:37:01,545 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => a80c4e623aab44bea3cfa33ee4fee8f3, NAME => 'TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:37:01,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => a80c4e623aab44bea3cfa33ee4fee8f3, NAME => 'TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:37:01,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a80c4e623aab44bea3cfa33ee4fee8f3, NAME => 'TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a80c4e623aab44bea3cfa33ee4fee8f3, NAME => 'TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:37:01,546 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/B/d8f232cdcdfd4c6c88d6f781bb680cb2 2024-11-17T15:37:01,572 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/C/8c15fb8fc2234ecc981a3a0f8016befb is 50, key is test_row_0/C:col10/1731857819885/Put/seqid=0 2024-11-17T15:37:01,582 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073741899_1075 (size=12301) 2024-11-17T15:37:01,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-17T15:37:01,698 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-17T15:37:01,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3. 2024-11-17T15:37:01,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3. as already flushing 2024-11-17T15:37:01,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3. 2024-11-17T15:37:01,698 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => a80c4e623aab44bea3cfa33ee4fee8f3, NAME => 'TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:37:01,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => a80c4e623aab44bea3cfa33ee4fee8f3, NAME => 'TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:37:01,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a80c4e623aab44bea3cfa33ee4fee8f3, NAME => 'TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a80c4e623aab44bea3cfa33ee4fee8f3, NAME => 'TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:37:01,798 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:01,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 213 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57522 deadline: 1731857881793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:01,799 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:01,799 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:01,799 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:37:01,852 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-17T15:37:01,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3. 2024-11-17T15:37:01,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3. as already flushing 2024-11-17T15:37:01,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3. 2024-11-17T15:37:01,852 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => a80c4e623aab44bea3cfa33ee4fee8f3, NAME => 'TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:37:01,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => a80c4e623aab44bea3cfa33ee4fee8f3, NAME => 'TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:37:01,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a80c4e623aab44bea3cfa33ee4fee8f3, NAME => 'TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a80c4e623aab44bea3cfa33ee4fee8f3, NAME => 'TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:37:01,985 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/C/8c15fb8fc2234ecc981a3a0f8016befb 2024-11-17T15:37:01,992 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/A/3429524979594aff8b5088c73a4644fb as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/A/3429524979594aff8b5088c73a4644fb 2024-11-17T15:37:02,004 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/A/3429524979594aff8b5088c73a4644fb, entries=150, sequenceid=293, filesize=12.0 K 2024-11-17T15:37:02,005 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/B/d8f232cdcdfd4c6c88d6f781bb680cb2 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/B/d8f232cdcdfd4c6c88d6f781bb680cb2 2024-11-17T15:37:02,008 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-17T15:37:02,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3. 2024-11-17T15:37:02,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3. as already flushing 2024-11-17T15:37:02,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3. 2024-11-17T15:37:02,008 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => a80c4e623aab44bea3cfa33ee4fee8f3, NAME => 'TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:37:02,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => a80c4e623aab44bea3cfa33ee4fee8f3, NAME => 'TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:37:02,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a80c4e623aab44bea3cfa33ee4fee8f3, NAME => 'TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a80c4e623aab44bea3cfa33ee4fee8f3, NAME => 'TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:37:02,012 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/B/d8f232cdcdfd4c6c88d6f781bb680cb2, entries=150, sequenceid=293, filesize=12.0 K 2024-11-17T15:37:02,017 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/C/8c15fb8fc2234ecc981a3a0f8016befb as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/C/8c15fb8fc2234ecc981a3a0f8016befb 2024-11-17T15:37:02,023 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/C/8c15fb8fc2234ecc981a3a0f8016befb, entries=150, sequenceid=293, filesize=12.0 K 2024-11-17T15:37:02,025 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for a80c4e623aab44bea3cfa33ee4fee8f3 in 962ms, sequenceid=293, compaction requested=true 2024-11-17T15:37:02,025 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a80c4e623aab44bea3cfa33ee4fee8f3: 2024-11-17T15:37:02,025 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a80c4e623aab44bea3cfa33ee4fee8f3:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T15:37:02,025 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T15:37:02,025 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T15:37:02,025 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a80c4e623aab44bea3cfa33ee4fee8f3:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T15:37:02,025 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T15:37:02,025 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a80c4e623aab44bea3cfa33ee4fee8f3:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T15:37:02,025 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-17T15:37:02,025 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T15:37:02,027 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37367 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T15:37:02,027 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37367 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T15:37:02,028 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1541): a80c4e623aab44bea3cfa33ee4fee8f3/A is initiating minor compaction (all files) 2024-11-17T15:37:02,028 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1541): a80c4e623aab44bea3cfa33ee4fee8f3/B is initiating minor compaction (all files) 2024-11-17T15:37:02,028 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a80c4e623aab44bea3cfa33ee4fee8f3/B in TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3. 2024-11-17T15:37:02,028 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/B/41c765f65c0a4c81bcfce4ca50c92d60, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/B/7b9e1176d8214ad8b9453f19e186a331, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/B/d8f232cdcdfd4c6c88d6f781bb680cb2] into tmpdir=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp, totalSize=36.5 K 2024-11-17T15:37:02,028 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a80c4e623aab44bea3cfa33ee4fee8f3/A in TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3. 2024-11-17T15:37:02,028 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/A/5c7e5227e0aa4248ad3d263081aae7de, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/A/dd4553c4191c4bbb81a5b45b9236645b, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/A/3429524979594aff8b5088c73a4644fb] into tmpdir=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp, totalSize=36.5 K 2024-11-17T15:37:02,028 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting 41c765f65c0a4c81bcfce4ca50c92d60, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1731857818314 2024-11-17T15:37:02,029 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5c7e5227e0aa4248ad3d263081aae7de, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1731857818314 2024-11-17T15:37:02,029 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting 7b9e1176d8214ad8b9453f19e186a331, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1731857819052 2024-11-17T15:37:02,029 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting dd4553c4191c4bbb81a5b45b9236645b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1731857819052 2024-11-17T15:37:02,030 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting d8f232cdcdfd4c6c88d6f781bb680cb2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1731857819885 2024-11-17T15:37:02,030 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3429524979594aff8b5088c73a4644fb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1731857819885 2024-11-17T15:37:02,043 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a80c4e623aab44bea3cfa33ee4fee8f3#B#compaction#63 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T15:37:02,043 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a80c4e623aab44bea3cfa33ee4fee8f3#A#compaction#64 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T15:37:02,044 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/A/a9d1135dc3a04ef2ad3ec9c701e6aec4 is 50, key is test_row_0/A:col10/1731857819885/Put/seqid=0 2024-11-17T15:37:02,044 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/B/de3bd57629c149a1af3b41574184fcef is 50, key is test_row_0/B:col10/1731857819885/Put/seqid=0 2024-11-17T15:37:02,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073741901_1077 (size=13017) 2024-11-17T15:37:02,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073741900_1076 (size=13017) 2024-11-17T15:37:02,162 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-17T15:37:02,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3. 2024-11-17T15:37:02,162 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing a80c4e623aab44bea3cfa33ee4fee8f3 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-17T15:37:02,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/A/7ecfbd90e83e48deae15512933781ac3 is 50, key is test_row_0/A:col10/1731857821165/Put/seqid=0 2024-11-17T15:37:02,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073741902_1078 (size=12301) 2024-11-17T15:37:02,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-17T15:37:02,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(8855): Flush requested on a80c4e623aab44bea3cfa33ee4fee8f3 2024-11-17T15:37:02,313 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3. as already flushing 2024-11-17T15:37:02,451 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:02,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 231 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57522 deadline: 1731857882448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:02,452 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:02,452 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:02,452 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:37:02,471 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/A/a9d1135dc3a04ef2ad3ec9c701e6aec4 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/A/a9d1135dc3a04ef2ad3ec9c701e6aec4 2024-11-17T15:37:02,482 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/B/de3bd57629c149a1af3b41574184fcef as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/B/de3bd57629c149a1af3b41574184fcef 2024-11-17T15:37:02,503 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a80c4e623aab44bea3cfa33ee4fee8f3/A of a80c4e623aab44bea3cfa33ee4fee8f3 into a9d1135dc3a04ef2ad3ec9c701e6aec4(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T15:37:02,503 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a80c4e623aab44bea3cfa33ee4fee8f3/B of a80c4e623aab44bea3cfa33ee4fee8f3 into de3bd57629c149a1af3b41574184fcef(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T15:37:02,503 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a80c4e623aab44bea3cfa33ee4fee8f3: 2024-11-17T15:37:02,503 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a80c4e623aab44bea3cfa33ee4fee8f3: 2024-11-17T15:37:02,503 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., storeName=a80c4e623aab44bea3cfa33ee4fee8f3/B, priority=13, startTime=1731857822025; duration=0sec 2024-11-17T15:37:02,503 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., storeName=a80c4e623aab44bea3cfa33ee4fee8f3/A, priority=13, startTime=1731857822025; duration=0sec 2024-11-17T15:37:02,503 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T15:37:02,503 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a80c4e623aab44bea3cfa33ee4fee8f3:A 2024-11-17T15:37:02,503 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T15:37:02,503 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a80c4e623aab44bea3cfa33ee4fee8f3:B 2024-11-17T15:37:02,503 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T15:37:02,505 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37367 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T15:37:02,505 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1541): a80c4e623aab44bea3cfa33ee4fee8f3/C is initiating minor compaction (all files) 2024-11-17T15:37:02,505 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a80c4e623aab44bea3cfa33ee4fee8f3/C in TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3. 2024-11-17T15:37:02,505 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/C/2f80a378a9834320b80026f1275588d3, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/C/59133d0cdf4344bfb2b222c734787150, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/C/8c15fb8fc2234ecc981a3a0f8016befb] into tmpdir=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp, totalSize=36.5 K 2024-11-17T15:37:02,506 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2f80a378a9834320b80026f1275588d3, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1731857818314 2024-11-17T15:37:02,506 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 59133d0cdf4344bfb2b222c734787150, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1731857819052 2024-11-17T15:37:02,507 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8c15fb8fc2234ecc981a3a0f8016befb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1731857819885 2024-11-17T15:37:02,525 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a80c4e623aab44bea3cfa33ee4fee8f3#C#compaction#66 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T15:37:02,526 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/C/d6703f4c50504d2986f5f53eab28b6f7 is 50, key is test_row_0/C:col10/1731857819885/Put/seqid=0 2024-11-17T15:37:02,564 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:02,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 232 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57522 deadline: 1731857882563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:02,565 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:02,565 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:02,565 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:37:02,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073741903_1079 (size=13017) 2024-11-17T15:37:02,585 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=309 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/A/7ecfbd90e83e48deae15512933781ac3 2024-11-17T15:37:02,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/B/6671c21f50444f3cbfc0e75479d64a76 is 50, key is test_row_0/B:col10/1731857821165/Put/seqid=0 2024-11-17T15:37:02,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073741904_1080 (size=12301) 2024-11-17T15:37:02,626 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=309 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/B/6671c21f50444f3cbfc0e75479d64a76 2024-11-17T15:37:02,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/C/0b3e1352b17747328564a6784658e4ae is 50, key is test_row_0/C:col10/1731857821165/Put/seqid=0 2024-11-17T15:37:02,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073741905_1081 (size=12301) 2024-11-17T15:37:02,664 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=309 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/C/0b3e1352b17747328564a6784658e4ae 2024-11-17T15:37:02,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/A/7ecfbd90e83e48deae15512933781ac3 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/A/7ecfbd90e83e48deae15512933781ac3 2024-11-17T15:37:02,685 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/A/7ecfbd90e83e48deae15512933781ac3, entries=150, sequenceid=309, filesize=12.0 K 2024-11-17T15:37:02,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/B/6671c21f50444f3cbfc0e75479d64a76 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/B/6671c21f50444f3cbfc0e75479d64a76 2024-11-17T15:37:02,697 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/B/6671c21f50444f3cbfc0e75479d64a76, entries=150, sequenceid=309, filesize=12.0 K 2024-11-17T15:37:02,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/C/0b3e1352b17747328564a6784658e4ae as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/C/0b3e1352b17747328564a6784658e4ae 2024-11-17T15:37:02,707 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/C/0b3e1352b17747328564a6784658e4ae, entries=150, sequenceid=309, filesize=12.0 K 2024-11-17T15:37:02,708 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for a80c4e623aab44bea3cfa33ee4fee8f3 in 546ms, sequenceid=309, compaction requested=false 2024-11-17T15:37:02,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for a80c4e623aab44bea3cfa33ee4fee8f3: 2024-11-17T15:37:02,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3. 2024-11-17T15:37:02,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-11-17T15:37:02,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-11-17T15:37:02,720 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-17T15:37:02,720 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6550 sec 2024-11-17T15:37:02,723 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 1.6650 sec 2024-11-17T15:37:02,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(8855): Flush requested on a80c4e623aab44bea3cfa33ee4fee8f3 2024-11-17T15:37:02,776 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T15:37:02,776 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a80c4e623aab44bea3cfa33ee4fee8f3 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-17T15:37:02,783 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/A/6593f7ff3eb247f298831d98d93c284d is 50, key is test_row_0/A:col10/1731857822413/Put/seqid=0 2024-11-17T15:37:02,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073741906_1082 (size=14741) 2024-11-17T15:37:02,811 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=332 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/A/6593f7ff3eb247f298831d98d93c284d 2024-11-17T15:37:02,840 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/B/eb4b6c3508384164a4d018fa91b50adf is 50, key is test_row_0/B:col10/1731857822413/Put/seqid=0 2024-11-17T15:37:02,872 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:02,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 246 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57522 deadline: 1731857882870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:02,873 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:02,873 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:02,873 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:37:02,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073741907_1083 (size=12301) 2024-11-17T15:37:02,886 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=332 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/B/eb4b6c3508384164a4d018fa91b50adf 2024-11-17T15:37:02,899 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/C/c3ab23d723ab4d028c6e2c788932b11a is 50, key is test_row_0/C:col10/1731857822413/Put/seqid=0 2024-11-17T15:37:02,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073741908_1084 (size=12301) 2024-11-17T15:37:02,990 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:02,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 247 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57522 deadline: 1731857882983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:02,991 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:02,991 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:02,991 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:37:02,993 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/C/d6703f4c50504d2986f5f53eab28b6f7 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/C/d6703f4c50504d2986f5f53eab28b6f7 2024-11-17T15:37:03,002 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a80c4e623aab44bea3cfa33ee4fee8f3/C of a80c4e623aab44bea3cfa33ee4fee8f3 into d6703f4c50504d2986f5f53eab28b6f7(size=12.7 K), total size for store is 24.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T15:37:03,002 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a80c4e623aab44bea3cfa33ee4fee8f3: 2024-11-17T15:37:03,002 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., storeName=a80c4e623aab44bea3cfa33ee4fee8f3/C, priority=13, startTime=1731857822025; duration=0sec 2024-11-17T15:37:03,002 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T15:37:03,002 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a80c4e623aab44bea3cfa33ee4fee8f3:C 2024-11-17T15:37:03,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-17T15:37:03,203 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T15:37:03,203 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:03,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 248 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57522 deadline: 1731857883203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:03,204 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T15:37:03,204 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:03,204 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:03,204 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:37:03,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-11-17T15:37:03,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-17T15:37:03,206 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T15:37:03,207 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T15:37:03,207 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T15:37:03,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-17T15:37:03,317 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=332 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/C/c3ab23d723ab4d028c6e2c788932b11a 2024-11-17T15:37:03,324 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/A/6593f7ff3eb247f298831d98d93c284d as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/A/6593f7ff3eb247f298831d98d93c284d 2024-11-17T15:37:03,331 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/A/6593f7ff3eb247f298831d98d93c284d, entries=200, sequenceid=332, filesize=14.4 K 2024-11-17T15:37:03,332 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/B/eb4b6c3508384164a4d018fa91b50adf as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/B/eb4b6c3508384164a4d018fa91b50adf 2024-11-17T15:37:03,340 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/B/eb4b6c3508384164a4d018fa91b50adf, entries=150, sequenceid=332, filesize=12.0 K 2024-11-17T15:37:03,342 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/C/c3ab23d723ab4d028c6e2c788932b11a as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/C/c3ab23d723ab4d028c6e2c788932b11a 2024-11-17T15:37:03,353 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/C/c3ab23d723ab4d028c6e2c788932b11a, entries=150, sequenceid=332, filesize=12.0 K 2024-11-17T15:37:03,354 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for a80c4e623aab44bea3cfa33ee4fee8f3 in 578ms, sequenceid=332, compaction requested=true 2024-11-17T15:37:03,354 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a80c4e623aab44bea3cfa33ee4fee8f3: 2024-11-17T15:37:03,355 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T15:37:03,355 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a80c4e623aab44bea3cfa33ee4fee8f3:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T15:37:03,355 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T15:37:03,355 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T15:37:03,356 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40059 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T15:37:03,356 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1541): a80c4e623aab44bea3cfa33ee4fee8f3/A is initiating minor compaction (all files) 2024-11-17T15:37:03,356 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a80c4e623aab44bea3cfa33ee4fee8f3/A in TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3. 2024-11-17T15:37:03,357 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/A/a9d1135dc3a04ef2ad3ec9c701e6aec4, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/A/7ecfbd90e83e48deae15512933781ac3, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/A/6593f7ff3eb247f298831d98d93c284d] into tmpdir=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp, totalSize=39.1 K 2024-11-17T15:37:03,357 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting a9d1135dc3a04ef2ad3ec9c701e6aec4, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1731857819885 2024-11-17T15:37:03,357 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T15:37:03,357 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a80c4e623aab44bea3cfa33ee4fee8f3:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T15:37:03,357 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1541): a80c4e623aab44bea3cfa33ee4fee8f3/B is initiating minor compaction (all files) 2024-11-17T15:37:03,358 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T15:37:03,358 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a80c4e623aab44bea3cfa33ee4fee8f3/B in TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3. 2024-11-17T15:37:03,358 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a80c4e623aab44bea3cfa33ee4fee8f3:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T15:37:03,358 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7ecfbd90e83e48deae15512933781ac3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1731857821148 2024-11-17T15:37:03,358 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T15:37:03,358 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/B/de3bd57629c149a1af3b41574184fcef, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/B/6671c21f50444f3cbfc0e75479d64a76, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/B/eb4b6c3508384164a4d018fa91b50adf] into tmpdir=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp, totalSize=36.7 K 2024-11-17T15:37:03,358 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting de3bd57629c149a1af3b41574184fcef, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1731857819885 2024-11-17T15:37:03,358 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6593f7ff3eb247f298831d98d93c284d, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1731857822398 2024-11-17T15:37:03,359 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting 6671c21f50444f3cbfc0e75479d64a76, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1731857821148 2024-11-17T15:37:03,360 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting eb4b6c3508384164a4d018fa91b50adf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1731857822398 2024-11-17T15:37:03,363 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-17T15:37:03,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3. 2024-11-17T15:37:03,363 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing a80c4e623aab44bea3cfa33ee4fee8f3 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-17T15:37:03,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/A/54db723928294532b7790f6c10d7d4bb is 50, key is test_row_0/A:col10/1731857822846/Put/seqid=0 2024-11-17T15:37:03,373 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a80c4e623aab44bea3cfa33ee4fee8f3#B#compaction#73 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T15:37:03,374 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/B/1174f57c6be549f4b24539c1e94ac2c9 is 50, key is test_row_0/B:col10/1731857822413/Put/seqid=0 2024-11-17T15:37:03,381 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a80c4e623aab44bea3cfa33ee4fee8f3#A#compaction#74 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T15:37:03,381 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/A/3433af5787874e2d87f4b8fb82f5ac5a is 50, key is test_row_0/A:col10/1731857822413/Put/seqid=0 2024-11-17T15:37:03,411 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073741910_1086 (size=12301) 2024-11-17T15:37:03,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073741909_1085 (size=13119) 2024-11-17T15:37:03,423 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/B/1174f57c6be549f4b24539c1e94ac2c9 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/B/1174f57c6be549f4b24539c1e94ac2c9 2024-11-17T15:37:03,442 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a80c4e623aab44bea3cfa33ee4fee8f3/B of a80c4e623aab44bea3cfa33ee4fee8f3 into 1174f57c6be549f4b24539c1e94ac2c9(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T15:37:03,442 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a80c4e623aab44bea3cfa33ee4fee8f3: 2024-11-17T15:37:03,442 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., storeName=a80c4e623aab44bea3cfa33ee4fee8f3/B, priority=13, startTime=1731857823355; duration=0sec 2024-11-17T15:37:03,442 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T15:37:03,442 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a80c4e623aab44bea3cfa33ee4fee8f3:B 2024-11-17T15:37:03,443 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T15:37:03,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073741911_1087 (size=13119) 2024-11-17T15:37:03,453 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T15:37:03,453 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1541): a80c4e623aab44bea3cfa33ee4fee8f3/C is initiating minor compaction (all files) 2024-11-17T15:37:03,453 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a80c4e623aab44bea3cfa33ee4fee8f3/C in TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3. 2024-11-17T15:37:03,453 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/C/d6703f4c50504d2986f5f53eab28b6f7, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/C/0b3e1352b17747328564a6784658e4ae, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/C/c3ab23d723ab4d028c6e2c788932b11a] into tmpdir=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp, totalSize=36.7 K 2024-11-17T15:37:03,454 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting d6703f4c50504d2986f5f53eab28b6f7, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1731857819885 2024-11-17T15:37:03,455 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting 0b3e1352b17747328564a6784658e4ae, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1731857821148 2024-11-17T15:37:03,456 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting c3ab23d723ab4d028c6e2c788932b11a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1731857822398 2024-11-17T15:37:03,486 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a80c4e623aab44bea3cfa33ee4fee8f3#C#compaction#75 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T15:37:03,486 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/C/53eafeb14b3344349776a7894042175b is 50, key is test_row_0/C:col10/1731857822413/Put/seqid=0 2024-11-17T15:37:03,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073741912_1088 (size=13119) 2024-11-17T15:37:03,520 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3. as already flushing 2024-11-17T15:37:03,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(8855): Flush requested on a80c4e623aab44bea3cfa33ee4fee8f3 2024-11-17T15:37:03,522 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/C/53eafeb14b3344349776a7894042175b as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/C/53eafeb14b3344349776a7894042175b 2024-11-17T15:37:03,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-17T15:37:03,539 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a80c4e623aab44bea3cfa33ee4fee8f3/C of a80c4e623aab44bea3cfa33ee4fee8f3 into 53eafeb14b3344349776a7894042175b(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T15:37:03,539 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a80c4e623aab44bea3cfa33ee4fee8f3: 2024-11-17T15:37:03,539 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., storeName=a80c4e623aab44bea3cfa33ee4fee8f3/C, priority=13, startTime=1731857823358; duration=0sec 2024-11-17T15:37:03,541 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T15:37:03,541 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a80c4e623aab44bea3cfa33ee4fee8f3:C 2024-11-17T15:37:03,659 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:03,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 267 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57522 deadline: 1731857883651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:03,660 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:03,660 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:03,660 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:37:03,763 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:03,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 268 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57522 deadline: 1731857883763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:03,764 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:03,764 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:03,764 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:37:03,813 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=348 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/A/54db723928294532b7790f6c10d7d4bb 2024-11-17T15:37:03,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-17T15:37:03,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/B/9ebb1e22e725441e9be046fdb8ca9246 is 50, key is test_row_0/B:col10/1731857822846/Put/seqid=0 2024-11-17T15:37:03,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073741913_1089 (size=12301) 2024-11-17T15:37:03,865 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/A/3433af5787874e2d87f4b8fb82f5ac5a as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/A/3433af5787874e2d87f4b8fb82f5ac5a 2024-11-17T15:37:03,872 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a80c4e623aab44bea3cfa33ee4fee8f3/A of a80c4e623aab44bea3cfa33ee4fee8f3 into 3433af5787874e2d87f4b8fb82f5ac5a(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T15:37:03,872 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a80c4e623aab44bea3cfa33ee4fee8f3: 2024-11-17T15:37:03,872 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., storeName=a80c4e623aab44bea3cfa33ee4fee8f3/A, priority=13, startTime=1731857823355; duration=0sec 2024-11-17T15:37:03,872 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T15:37:03,872 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a80c4e623aab44bea3cfa33ee4fee8f3:A 2024-11-17T15:37:03,976 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:03,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 269 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57522 deadline: 1731857883974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:03,977 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:03,977 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:03,977 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:37:04,249 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=348 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/B/9ebb1e22e725441e9be046fdb8ca9246 2024-11-17T15:37:04,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/C/a492ff331e81468f913c445932ccc249 is 50, key is test_row_0/C:col10/1731857822846/Put/seqid=0 2024-11-17T15:37:04,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073741914_1090 (size=12301) 2024-11-17T15:37:04,284 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:04,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 270 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57522 deadline: 1731857884283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:04,286 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:04,286 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:04,286 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:37:04,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-17T15:37:04,673 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=348 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/C/a492ff331e81468f913c445932ccc249 2024-11-17T15:37:04,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/A/54db723928294532b7790f6c10d7d4bb as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/A/54db723928294532b7790f6c10d7d4bb 2024-11-17T15:37:04,702 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/A/54db723928294532b7790f6c10d7d4bb, entries=150, sequenceid=348, filesize=12.0 K 2024-11-17T15:37:04,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/B/9ebb1e22e725441e9be046fdb8ca9246 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/B/9ebb1e22e725441e9be046fdb8ca9246 2024-11-17T15:37:04,710 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/B/9ebb1e22e725441e9be046fdb8ca9246, entries=150, sequenceid=348, filesize=12.0 K 2024-11-17T15:37:04,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/C/a492ff331e81468f913c445932ccc249 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/C/a492ff331e81468f913c445932ccc249 2024-11-17T15:37:04,718 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/C/a492ff331e81468f913c445932ccc249, entries=150, sequenceid=348, filesize=12.0 K 2024-11-17T15:37:04,719 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for a80c4e623aab44bea3cfa33ee4fee8f3 in 1356ms, sequenceid=348, compaction requested=false 2024-11-17T15:37:04,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for a80c4e623aab44bea3cfa33ee4fee8f3: 2024-11-17T15:37:04,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3. 2024-11-17T15:37:04,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-11-17T15:37:04,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-11-17T15:37:04,725 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-11-17T15:37:04,725 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5150 sec 2024-11-17T15:37:04,729 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 1.5220 sec 2024-11-17T15:37:04,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(8855): Flush requested on a80c4e623aab44bea3cfa33ee4fee8f3 2024-11-17T15:37:04,798 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T15:37:04,799 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a80c4e623aab44bea3cfa33ee4fee8f3 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-17T15:37:04,805 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/A/9d5b07bab477452f9c268f8993b0c465 is 50, key is test_row_0/A:col10/1731857823632/Put/seqid=0 2024-11-17T15:37:04,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073741915_1091 (size=14741) 2024-11-17T15:37:04,813 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=373 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/A/9d5b07bab477452f9c268f8993b0c465 2024-11-17T15:37:04,830 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/B/654f869346f2457d9fdbf7b3bcf7e61e is 50, key is test_row_0/B:col10/1731857823632/Put/seqid=0 2024-11-17T15:37:04,876 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073741916_1092 (size=12301) 2024-11-17T15:37:04,876 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:04,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 283 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57522 deadline: 1731857884871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:04,877 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:04,877 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:04,877 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:37:04,878 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=373 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/B/654f869346f2457d9fdbf7b3bcf7e61e 2024-11-17T15:37:04,901 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/C/a670e04ed9b541fea8191a23c27fbfda is 50, key is test_row_0/C:col10/1731857823632/Put/seqid=0 2024-11-17T15:37:04,907 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073741917_1093 (size=12301) 2024-11-17T15:37:04,986 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:04,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 284 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57522 deadline: 1731857884983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:04,987 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:04,987 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:04,987 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:37:05,196 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:05,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 285 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57522 deadline: 1731857885193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:05,197 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:05,197 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:05,197 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:37:05,308 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=373 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/C/a670e04ed9b541fea8191a23c27fbfda 2024-11-17T15:37:05,320 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/A/9d5b07bab477452f9c268f8993b0c465 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/A/9d5b07bab477452f9c268f8993b0c465 2024-11-17T15:37:05,327 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/A/9d5b07bab477452f9c268f8993b0c465, entries=200, sequenceid=373, filesize=14.4 K 2024-11-17T15:37:05,328 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/B/654f869346f2457d9fdbf7b3bcf7e61e as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/B/654f869346f2457d9fdbf7b3bcf7e61e 2024-11-17T15:37:05,334 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/B/654f869346f2457d9fdbf7b3bcf7e61e, entries=150, sequenceid=373, filesize=12.0 K 2024-11-17T15:37:05,337 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/C/a670e04ed9b541fea8191a23c27fbfda as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/C/a670e04ed9b541fea8191a23c27fbfda 2024-11-17T15:37:05,343 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/C/a670e04ed9b541fea8191a23c27fbfda, entries=150, sequenceid=373, filesize=12.0 K 2024-11-17T15:37:05,345 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for a80c4e623aab44bea3cfa33ee4fee8f3 in 547ms, sequenceid=373, compaction requested=true 2024-11-17T15:37:05,345 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a80c4e623aab44bea3cfa33ee4fee8f3: 2024-11-17T15:37:05,345 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a80c4e623aab44bea3cfa33ee4fee8f3:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T15:37:05,345 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T15:37:05,345 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T15:37:05,345 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a80c4e623aab44bea3cfa33ee4fee8f3:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T15:37:05,345 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T15:37:05,346 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a80c4e623aab44bea3cfa33ee4fee8f3:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T15:37:05,346 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T15:37:05,346 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T15:37:05,353 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40161 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T15:37:05,353 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T15:37:05,353 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1541): a80c4e623aab44bea3cfa33ee4fee8f3/B is initiating minor compaction (all files) 2024-11-17T15:37:05,353 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1541): a80c4e623aab44bea3cfa33ee4fee8f3/A is initiating minor compaction (all files) 2024-11-17T15:37:05,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-17T15:37:05,353 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a80c4e623aab44bea3cfa33ee4fee8f3/B in TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3. 2024-11-17T15:37:05,353 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a80c4e623aab44bea3cfa33ee4fee8f3/A in TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3. 2024-11-17T15:37:05,353 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/B/1174f57c6be549f4b24539c1e94ac2c9, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/B/9ebb1e22e725441e9be046fdb8ca9246, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/B/654f869346f2457d9fdbf7b3bcf7e61e] into tmpdir=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp, totalSize=36.8 K 2024-11-17T15:37:05,354 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/A/3433af5787874e2d87f4b8fb82f5ac5a, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/A/54db723928294532b7790f6c10d7d4bb, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/A/9d5b07bab477452f9c268f8993b0c465] into tmpdir=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp, totalSize=39.2 K 2024-11-17T15:37:05,354 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T15:37:05,354 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3433af5787874e2d87f4b8fb82f5ac5a, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1731857822398 2024-11-17T15:37:05,354 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting 1174f57c6be549f4b24539c1e94ac2c9, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1731857822398 2024-11-17T15:37:05,354 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T15:37:05,355 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting 9ebb1e22e725441e9be046fdb8ca9246, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=348, earliestPutTs=1731857822820 2024-11-17T15:37:05,355 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting 654f869346f2457d9fdbf7b3bcf7e61e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=373, earliestPutTs=1731857823632 2024-11-17T15:37:05,355 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 54db723928294532b7790f6c10d7d4bb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=348, earliestPutTs=1731857822820 2024-11-17T15:37:05,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees 2024-11-17T15:37:05,357 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9d5b07bab477452f9c268f8993b0c465, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=373, earliestPutTs=1731857823632 2024-11-17T15:37:05,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-17T15:37:05,358 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T15:37:05,359 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T15:37:05,359 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T15:37:05,371 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a80c4e623aab44bea3cfa33ee4fee8f3#B#compaction#81 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T15:37:05,372 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/B/2078b0b2034c46fa95fb475c1dcb139d is 50, key is test_row_0/B:col10/1731857823632/Put/seqid=0 2024-11-17T15:37:05,382 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a80c4e623aab44bea3cfa33ee4fee8f3#A#compaction#82 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T15:37:05,383 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/A/648fa15c4f8743619ac526b13715f378 is 50, key is test_row_0/A:col10/1731857823632/Put/seqid=0 2024-11-17T15:37:05,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073741918_1094 (size=13221) 2024-11-17T15:37:05,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073741919_1095 (size=13221) 2024-11-17T15:37:05,415 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/B/2078b0b2034c46fa95fb475c1dcb139d as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/B/2078b0b2034c46fa95fb475c1dcb139d 2024-11-17T15:37:05,415 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/A/648fa15c4f8743619ac526b13715f378 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/A/648fa15c4f8743619ac526b13715f378 2024-11-17T15:37:05,423 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a80c4e623aab44bea3cfa33ee4fee8f3/B of a80c4e623aab44bea3cfa33ee4fee8f3 into 2078b0b2034c46fa95fb475c1dcb139d(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T15:37:05,423 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a80c4e623aab44bea3cfa33ee4fee8f3: 2024-11-17T15:37:05,423 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., storeName=a80c4e623aab44bea3cfa33ee4fee8f3/B, priority=13, startTime=1731857825345; duration=0sec 2024-11-17T15:37:05,423 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a80c4e623aab44bea3cfa33ee4fee8f3/A of a80c4e623aab44bea3cfa33ee4fee8f3 into 648fa15c4f8743619ac526b13715f378(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T15:37:05,424 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a80c4e623aab44bea3cfa33ee4fee8f3: 2024-11-17T15:37:05,424 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., storeName=a80c4e623aab44bea3cfa33ee4fee8f3/A, priority=13, startTime=1731857825345; duration=0sec 2024-11-17T15:37:05,424 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T15:37:05,424 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a80c4e623aab44bea3cfa33ee4fee8f3:B 2024-11-17T15:37:05,424 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T15:37:05,424 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a80c4e623aab44bea3cfa33ee4fee8f3:A 2024-11-17T15:37:05,424 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T15:37:05,425 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T15:37:05,426 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1541): a80c4e623aab44bea3cfa33ee4fee8f3/C is initiating minor compaction (all files) 2024-11-17T15:37:05,426 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a80c4e623aab44bea3cfa33ee4fee8f3/C in TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3. 2024-11-17T15:37:05,426 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/C/53eafeb14b3344349776a7894042175b, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/C/a492ff331e81468f913c445932ccc249, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/C/a670e04ed9b541fea8191a23c27fbfda] into tmpdir=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp, totalSize=36.8 K 2024-11-17T15:37:05,426 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting 53eafeb14b3344349776a7894042175b, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1731857822398 2024-11-17T15:37:05,427 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting a492ff331e81468f913c445932ccc249, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=348, earliestPutTs=1731857822820 2024-11-17T15:37:05,427 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting a670e04ed9b541fea8191a23c27fbfda, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=373, earliestPutTs=1731857823632 2024-11-17T15:37:05,436 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a80c4e623aab44bea3cfa33ee4fee8f3#C#compaction#83 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T15:37:05,437 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/C/2d7a58ef8c504d50a868234da5bfb99e is 50, key is test_row_0/C:col10/1731857823632/Put/seqid=0 2024-11-17T15:37:05,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073741920_1096 (size=13221) 2024-11-17T15:37:05,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-17T15:37:05,512 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-17T15:37:05,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3. 2024-11-17T15:37:05,512 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2902): Flushing a80c4e623aab44bea3cfa33ee4fee8f3 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-17T15:37:05,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(8855): Flush requested on a80c4e623aab44bea3cfa33ee4fee8f3 2024-11-17T15:37:05,520 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3. as already flushing 2024-11-17T15:37:05,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/A/673ac698a90847c2a07f220e762aeae5 is 50, key is test_row_0/A:col10/1731857824861/Put/seqid=0 2024-11-17T15:37:05,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073741921_1097 (size=12301) 2024-11-17T15:37:05,578 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=390 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/A/673ac698a90847c2a07f220e762aeae5 2024-11-17T15:37:05,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/B/19e2b263a1bf443f824ee0240dc0ccc1 is 50, key is test_row_0/B:col10/1731857824861/Put/seqid=0 2024-11-17T15:37:05,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073741922_1098 (size=12301) 2024-11-17T15:37:05,622 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=390 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/B/19e2b263a1bf443f824ee0240dc0ccc1 2024-11-17T15:37:05,637 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:05,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/C/ef8241739fc04978a4a6cd7527470727 is 50, key is test_row_0/C:col10/1731857824861/Put/seqid=0 2024-11-17T15:37:05,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 305 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57522 deadline: 1731857885636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:05,639 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:05,639 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:05,639 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:37:05,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073741923_1099 (size=12301) 2024-11-17T15:37:05,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-17T15:37:05,746 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:05,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 306 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57522 deadline: 1731857885743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:05,747 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:05,747 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:05,747 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:37:05,857 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/C/2d7a58ef8c504d50a868234da5bfb99e as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/C/2d7a58ef8c504d50a868234da5bfb99e 2024-11-17T15:37:05,866 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a80c4e623aab44bea3cfa33ee4fee8f3/C of a80c4e623aab44bea3cfa33ee4fee8f3 into 2d7a58ef8c504d50a868234da5bfb99e(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T15:37:05,866 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a80c4e623aab44bea3cfa33ee4fee8f3: 2024-11-17T15:37:05,866 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., storeName=a80c4e623aab44bea3cfa33ee4fee8f3/C, priority=13, startTime=1731857825346; duration=0sec 2024-11-17T15:37:05,866 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T15:37:05,866 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a80c4e623aab44bea3cfa33ee4fee8f3:C 2024-11-17T15:37:05,956 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:05,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 307 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57522 deadline: 1731857885953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:05,957 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:05,957 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:05,958 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:37:05,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-17T15:37:06,058 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=390 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/C/ef8241739fc04978a4a6cd7527470727 2024-11-17T15:37:06,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/A/673ac698a90847c2a07f220e762aeae5 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/A/673ac698a90847c2a07f220e762aeae5 2024-11-17T15:37:06,072 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/A/673ac698a90847c2a07f220e762aeae5, entries=150, sequenceid=390, filesize=12.0 K 2024-11-17T15:37:06,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/B/19e2b263a1bf443f824ee0240dc0ccc1 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/B/19e2b263a1bf443f824ee0240dc0ccc1 2024-11-17T15:37:06,079 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/B/19e2b263a1bf443f824ee0240dc0ccc1, entries=150, sequenceid=390, filesize=12.0 K 2024-11-17T15:37:06,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/C/ef8241739fc04978a4a6cd7527470727 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/C/ef8241739fc04978a4a6cd7527470727 2024-11-17T15:37:06,086 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/C/ef8241739fc04978a4a6cd7527470727, entries=150, sequenceid=390, filesize=12.0 K 2024-11-17T15:37:06,087 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for a80c4e623aab44bea3cfa33ee4fee8f3 in 575ms, sequenceid=390, compaction requested=false 2024-11-17T15:37:06,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2603): Flush status journal for a80c4e623aab44bea3cfa33ee4fee8f3: 2024-11-17T15:37:06,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3. 2024-11-17T15:37:06,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=24 2024-11-17T15:37:06,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.HMaster(4169): Remote procedure done, pid=24 2024-11-17T15:37:06,093 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-17T15:37:06,093 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 730 msec 2024-11-17T15:37:06,095 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees in 739 msec 2024-11-17T15:37:06,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(8855): Flush requested on a80c4e623aab44bea3cfa33ee4fee8f3 2024-11-17T15:37:06,268 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T15:37:06,268 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a80c4e623aab44bea3cfa33ee4fee8f3 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-17T15:37:06,277 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/A/4a5f41439ffe471f94f3f0b7cf34ce80 is 50, key is test_row_0/A:col10/1731857825617/Put/seqid=0 2024-11-17T15:37:06,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073741924_1100 (size=12301) 2024-11-17T15:37:06,323 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:06,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 320 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57522 deadline: 1731857886323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:06,324 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:06,324 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:06,324 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:37:06,436 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:06,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 321 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57522 deadline: 1731857886433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:06,437 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:06,437 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:06,437 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:37:06,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-17T15:37:06,493 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T15:37:06,494 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T15:37:06,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees 2024-11-17T15:37:06,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-17T15:37:06,496 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T15:37:06,498 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T15:37:06,498 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T15:37:06,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-17T15:37:06,643 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:06,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 322 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57522 deadline: 1731857886643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:06,644 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:06,644 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:06,644 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:37:06,651 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-17T15:37:06,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3. 2024-11-17T15:37:06,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3. as already flushing 2024-11-17T15:37:06,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3. 2024-11-17T15:37:06,652 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => a80c4e623aab44bea3cfa33ee4fee8f3, NAME => 'TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:37:06,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => a80c4e623aab44bea3cfa33ee4fee8f3, NAME => 'TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:37:06,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a80c4e623aab44bea3cfa33ee4fee8f3, NAME => 'TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a80c4e623aab44bea3cfa33ee4fee8f3, NAME => 'TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:37:06,684 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=413 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/A/4a5f41439ffe471f94f3f0b7cf34ce80 2024-11-17T15:37:06,696 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/B/4f2351093f8840dcb9f93a56fe54793f is 50, key is test_row_0/B:col10/1731857825617/Put/seqid=0 2024-11-17T15:37:06,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073741925_1101 (size=12301) 2024-11-17T15:37:06,710 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=413 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/B/4f2351093f8840dcb9f93a56fe54793f 2024-11-17T15:37:06,728 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/C/681188ae0d054188b15c451ba61e8e7d is 50, key is test_row_0/C:col10/1731857825617/Put/seqid=0 2024-11-17T15:37:06,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073741926_1102 (size=12301) 2024-11-17T15:37:06,810 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-17T15:37:06,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3. 2024-11-17T15:37:06,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3. as already flushing 2024-11-17T15:37:06,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3. 2024-11-17T15:37:06,811 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => a80c4e623aab44bea3cfa33ee4fee8f3, NAME => 'TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:37:06,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => a80c4e623aab44bea3cfa33ee4fee8f3, NAME => 'TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:37:06,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a80c4e623aab44bea3cfa33ee4fee8f3, NAME => 'TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a80c4e623aab44bea3cfa33ee4fee8f3, NAME => 'TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:37:06,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-17T15:37:06,953 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:06,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 323 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57522 deadline: 1731857886953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:06,954 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:06,954 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:06,954 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:37:06,965 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-17T15:37:06,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3. 2024-11-17T15:37:06,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3. as already flushing 2024-11-17T15:37:06,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3. 2024-11-17T15:37:06,965 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => a80c4e623aab44bea3cfa33ee4fee8f3, NAME => 'TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:37:06,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => a80c4e623aab44bea3cfa33ee4fee8f3, NAME => 'TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:37:06,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a80c4e623aab44bea3cfa33ee4fee8f3, NAME => 'TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a80c4e623aab44bea3cfa33ee4fee8f3, NAME => 'TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:37:07,062 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-11-17T15:37:07,062 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T15:37:07,063 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-11-17T15:37:07,063 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T15:37:07,065 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-11-17T15:37:07,065 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T15:37:07,066 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-11-17T15:37:07,066 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T15:37:07,067 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:37:07,067 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:37:07,068 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:37:07,068 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:37:07,068 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:37:07,068 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:37:07,068 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T15:37:07,068 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T15:37:07,068 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T15:37:07,069 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:37:07,069 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:37:07,069 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T15:37:07,119 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-17T15:37:07,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3. 2024-11-17T15:37:07,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3. as already flushing 2024-11-17T15:37:07,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3. 2024-11-17T15:37:07,120 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => a80c4e623aab44bea3cfa33ee4fee8f3, NAME => 'TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:37:07,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => a80c4e623aab44bea3cfa33ee4fee8f3, NAME => 'TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:37:07,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a80c4e623aab44bea3cfa33ee4fee8f3, NAME => 'TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a80c4e623aab44bea3cfa33ee4fee8f3, NAME => 'TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:37:07,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-17T15:37:07,135 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=413 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/C/681188ae0d054188b15c451ba61e8e7d 2024-11-17T15:37:07,141 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/A/4a5f41439ffe471f94f3f0b7cf34ce80 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/A/4a5f41439ffe471f94f3f0b7cf34ce80 2024-11-17T15:37:07,146 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/A/4a5f41439ffe471f94f3f0b7cf34ce80, entries=150, sequenceid=413, filesize=12.0 K 2024-11-17T15:37:07,147 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/B/4f2351093f8840dcb9f93a56fe54793f as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/B/4f2351093f8840dcb9f93a56fe54793f 2024-11-17T15:37:07,153 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/B/4f2351093f8840dcb9f93a56fe54793f, entries=150, sequenceid=413, filesize=12.0 K 2024-11-17T15:37:07,153 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/C/681188ae0d054188b15c451ba61e8e7d as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/C/681188ae0d054188b15c451ba61e8e7d 2024-11-17T15:37:07,158 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/C/681188ae0d054188b15c451ba61e8e7d, entries=150, sequenceid=413, filesize=12.0 K 2024-11-17T15:37:07,159 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for a80c4e623aab44bea3cfa33ee4fee8f3 in 891ms, sequenceid=413, compaction requested=true 2024-11-17T15:37:07,159 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a80c4e623aab44bea3cfa33ee4fee8f3: 2024-11-17T15:37:07,159 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a80c4e623aab44bea3cfa33ee4fee8f3:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T15:37:07,160 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T15:37:07,160 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a80c4e623aab44bea3cfa33ee4fee8f3:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T15:37:07,160 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T15:37:07,160 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a80c4e623aab44bea3cfa33ee4fee8f3:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T15:37:07,160 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T15:37:07,160 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T15:37:07,160 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T15:37:07,161 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T15:37:07,161 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T15:37:07,161 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1541): a80c4e623aab44bea3cfa33ee4fee8f3/A is initiating minor compaction (all files) 2024-11-17T15:37:07,161 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1541): a80c4e623aab44bea3cfa33ee4fee8f3/B is initiating minor compaction (all files) 2024-11-17T15:37:07,161 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a80c4e623aab44bea3cfa33ee4fee8f3/A in TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3. 2024-11-17T15:37:07,161 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a80c4e623aab44bea3cfa33ee4fee8f3/B in TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3. 2024-11-17T15:37:07,161 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/A/648fa15c4f8743619ac526b13715f378, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/A/673ac698a90847c2a07f220e762aeae5, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/A/4a5f41439ffe471f94f3f0b7cf34ce80] into tmpdir=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp, totalSize=36.9 K 2024-11-17T15:37:07,161 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/B/2078b0b2034c46fa95fb475c1dcb139d, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/B/19e2b263a1bf443f824ee0240dc0ccc1, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/B/4f2351093f8840dcb9f93a56fe54793f] into tmpdir=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp, totalSize=36.9 K 2024-11-17T15:37:07,162 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 648fa15c4f8743619ac526b13715f378, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=373, earliestPutTs=1731857823632 2024-11-17T15:37:07,162 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting 2078b0b2034c46fa95fb475c1dcb139d, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=373, earliestPutTs=1731857823632 2024-11-17T15:37:07,162 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting 19e2b263a1bf443f824ee0240dc0ccc1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=390, earliestPutTs=1731857824840 2024-11-17T15:37:07,162 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 673ac698a90847c2a07f220e762aeae5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=390, earliestPutTs=1731857824840 2024-11-17T15:37:07,162 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting 4f2351093f8840dcb9f93a56fe54793f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=413, earliestPutTs=1731857825605 2024-11-17T15:37:07,162 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4a5f41439ffe471f94f3f0b7cf34ce80, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=413, earliestPutTs=1731857825605 2024-11-17T15:37:07,171 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a80c4e623aab44bea3cfa33ee4fee8f3#B#compaction#91 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T15:37:07,171 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a80c4e623aab44bea3cfa33ee4fee8f3#A#compaction#90 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T15:37:07,172 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/B/42742a62c6a24f109f1bde0fabc13d69 is 50, key is test_row_0/B:col10/1731857825617/Put/seqid=0 2024-11-17T15:37:07,172 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/A/13a92a9f13f844ef944ddff7f62d806a is 50, key is test_row_0/A:col10/1731857825617/Put/seqid=0 2024-11-17T15:37:07,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073741927_1103 (size=13323) 2024-11-17T15:37:07,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073741928_1104 (size=13323) 2024-11-17T15:37:07,273 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-17T15:37:07,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3. 2024-11-17T15:37:07,274 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2902): Flushing a80c4e623aab44bea3cfa33ee4fee8f3 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-17T15:37:07,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/A/4e7c6202aab5454ba2c2b30341ee8416 is 50, key is test_row_0/A:col10/1731857826316/Put/seqid=0 2024-11-17T15:37:07,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073741929_1105 (size=12301) 2024-11-17T15:37:07,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(8855): Flush requested on a80c4e623aab44bea3cfa33ee4fee8f3 2024-11-17T15:37:07,464 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3. as already flushing 2024-11-17T15:37:07,465 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-11-17T15:37:07,465 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T15:37:07,465 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:37:07,465 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:37:07,465 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T15:37:07,590 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/A/13a92a9f13f844ef944ddff7f62d806a as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/A/13a92a9f13f844ef944ddff7f62d806a 2024-11-17T15:37:07,590 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/B/42742a62c6a24f109f1bde0fabc13d69 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/B/42742a62c6a24f109f1bde0fabc13d69 2024-11-17T15:37:07,595 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a80c4e623aab44bea3cfa33ee4fee8f3/A of a80c4e623aab44bea3cfa33ee4fee8f3 into 13a92a9f13f844ef944ddff7f62d806a(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T15:37:07,595 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a80c4e623aab44bea3cfa33ee4fee8f3: 2024-11-17T15:37:07,595 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., storeName=a80c4e623aab44bea3cfa33ee4fee8f3/A, priority=13, startTime=1731857827159; duration=0sec 2024-11-17T15:37:07,595 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T15:37:07,595 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a80c4e623aab44bea3cfa33ee4fee8f3:A 2024-11-17T15:37:07,596 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a80c4e623aab44bea3cfa33ee4fee8f3/B of a80c4e623aab44bea3cfa33ee4fee8f3 into 42742a62c6a24f109f1bde0fabc13d69(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T15:37:07,596 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T15:37:07,596 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a80c4e623aab44bea3cfa33ee4fee8f3: 2024-11-17T15:37:07,596 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., storeName=a80c4e623aab44bea3cfa33ee4fee8f3/B, priority=13, startTime=1731857827160; duration=0sec 2024-11-17T15:37:07,596 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T15:37:07,596 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a80c4e623aab44bea3cfa33ee4fee8f3:B 2024-11-17T15:37:07,597 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T15:37:07,597 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1541): a80c4e623aab44bea3cfa33ee4fee8f3/C is initiating minor compaction (all files) 2024-11-17T15:37:07,597 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a80c4e623aab44bea3cfa33ee4fee8f3/C in TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3. 2024-11-17T15:37:07,597 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/C/2d7a58ef8c504d50a868234da5bfb99e, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/C/ef8241739fc04978a4a6cd7527470727, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/C/681188ae0d054188b15c451ba61e8e7d] into tmpdir=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp, totalSize=36.9 K 2024-11-17T15:37:07,597 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2d7a58ef8c504d50a868234da5bfb99e, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=373, earliestPutTs=1731857823632 2024-11-17T15:37:07,598 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting ef8241739fc04978a4a6cd7527470727, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=390, earliestPutTs=1731857824840 2024-11-17T15:37:07,598 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 681188ae0d054188b15c451ba61e8e7d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=413, earliestPutTs=1731857825605 2024-11-17T15:37:07,605 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a80c4e623aab44bea3cfa33ee4fee8f3#C#compaction#93 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T15:37:07,606 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/C/4740d0bfb38942809bc7cc5c26a3a155 is 50, key is test_row_0/C:col10/1731857825617/Put/seqid=0 2024-11-17T15:37:07,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073741930_1106 (size=13323) 2024-11-17T15:37:07,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-17T15:37:07,685 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=427 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/A/4e7c6202aab5454ba2c2b30341ee8416 2024-11-17T15:37:07,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/B/0cbd9ce0277c4186bcc7410decd2118e is 50, key is test_row_0/B:col10/1731857826316/Put/seqid=0 2024-11-17T15:37:07,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073741931_1107 (size=12301) 2024-11-17T15:37:08,016 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/C/4740d0bfb38942809bc7cc5c26a3a155 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/C/4740d0bfb38942809bc7cc5c26a3a155 2024-11-17T15:37:08,021 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a80c4e623aab44bea3cfa33ee4fee8f3/C of a80c4e623aab44bea3cfa33ee4fee8f3 into 4740d0bfb38942809bc7cc5c26a3a155(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T15:37:08,021 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a80c4e623aab44bea3cfa33ee4fee8f3: 2024-11-17T15:37:08,021 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., storeName=a80c4e623aab44bea3cfa33ee4fee8f3/C, priority=13, startTime=1731857827160; duration=0sec 2024-11-17T15:37:08,021 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T15:37:08,021 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a80c4e623aab44bea3cfa33ee4fee8f3:C 2024-11-17T15:37:08,098 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=427 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/B/0cbd9ce0277c4186bcc7410decd2118e 2024-11-17T15:37:08,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/C/7420f113336f4966a5e53be569edce53 is 50, key is test_row_0/C:col10/1731857826316/Put/seqid=0 2024-11-17T15:37:08,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073741932_1108 (size=12301) 2024-11-17T15:37:08,511 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=427 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/C/7420f113336f4966a5e53be569edce53 2024-11-17T15:37:08,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/A/4e7c6202aab5454ba2c2b30341ee8416 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/A/4e7c6202aab5454ba2c2b30341ee8416 2024-11-17T15:37:08,521 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/A/4e7c6202aab5454ba2c2b30341ee8416, entries=150, sequenceid=427, filesize=12.0 K 2024-11-17T15:37:08,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/B/0cbd9ce0277c4186bcc7410decd2118e as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/B/0cbd9ce0277c4186bcc7410decd2118e 2024-11-17T15:37:08,526 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/B/0cbd9ce0277c4186bcc7410decd2118e, entries=150, sequenceid=427, filesize=12.0 K 2024-11-17T15:37:08,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/C/7420f113336f4966a5e53be569edce53 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/C/7420f113336f4966a5e53be569edce53 2024-11-17T15:37:08,532 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/C/7420f113336f4966a5e53be569edce53, entries=150, sequenceid=427, filesize=12.0 K 2024-11-17T15:37:08,533 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=6.71 KB/6870 for a80c4e623aab44bea3cfa33ee4fee8f3 in 1259ms, sequenceid=427, compaction requested=false 2024-11-17T15:37:08,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2603): Flush status journal for a80c4e623aab44bea3cfa33ee4fee8f3: 2024-11-17T15:37:08,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3. 2024-11-17T15:37:08,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=26 2024-11-17T15:37:08,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.HMaster(4169): Remote procedure done, pid=26 2024-11-17T15:37:08,537 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-17T15:37:08,537 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0370 sec 2024-11-17T15:37:08,539 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees in 2.0440 sec 2024-11-17T15:37:08,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-17T15:37:08,644 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T15:37:09,090 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-17T15:37:10,385 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-11-17T15:37:10,385 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T15:37:10,385 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:37:10,385 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:37:10,385 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T15:37:10,424 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-11-17T15:37:10,424 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T15:37:10,424 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:37:10,424 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:37:10,424 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T15:37:10,444 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-11-17T15:37:10,444 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T15:37:10,444 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:37:10,444 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:37:10,444 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T15:37:10,475 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-11-17T15:37:10,475 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T15:37:10,475 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:37:10,475 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:37:10,475 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T15:37:10,475 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-17T15:37:10,476 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 253 2024-11-17T15:37:10,476 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 20 2024-11-17T15:37:10,476 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 22 2024-11-17T15:37:10,476 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 21 2024-11-17T15:37:10,476 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 22 2024-11-17T15:37:10,476 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-17T15:37:10,476 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2762 2024-11-17T15:37:10,476 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2768 2024-11-17T15:37:10,476 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-17T15:37:10,476 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1264 2024-11-17T15:37:10,476 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3786 rows 2024-11-17T15:37:10,476 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1265 2024-11-17T15:37:10,476 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3791 rows 2024-11-17T15:37:10,477 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-17T15:37:10,477 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-17T15:37:10,477 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:37:10,477 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:37:10,477 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T15:37:10,482 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-17T15:37:10,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] procedure2.ProcedureExecutor(1139): Stored pid=27, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-17T15:37:10,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-17T15:37:10,490 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731857830490"}]},"ts":"1731857830490"} 2024-11-17T15:37:10,492 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-17T15:37:10,492 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-17T15:37:10,494 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-17T15:37:10,498 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a80c4e623aab44bea3cfa33ee4fee8f3, UNASSIGN}] 2024-11-17T15:37:10,499 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a80c4e623aab44bea3cfa33ee4fee8f3, UNASSIGN 2024-11-17T15:37:10,500 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=a80c4e623aab44bea3cfa33ee4fee8f3, regionState=CLOSING, regionLocation=7a780d55532c,32955,1731857800648 2024-11-17T15:37:10,502 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a80c4e623aab44bea3cfa33ee4fee8f3, UNASSIGN because future has completed 2024-11-17T15:37:10,502 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-17T15:37:10,503 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE, hasLock=false; CloseRegionProcedure a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648}] 2024-11-17T15:37:10,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-17T15:37:10,659 INFO [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(122): Close a80c4e623aab44bea3cfa33ee4fee8f3 2024-11-17T15:37:10,660 DEBUG [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-17T15:37:10,660 DEBUG [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1722): Closing a80c4e623aab44bea3cfa33ee4fee8f3, disabling compactions & flushes 2024-11-17T15:37:10,661 INFO [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3. 2024-11-17T15:37:10,661 DEBUG [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3. 2024-11-17T15:37:10,661 DEBUG [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3. after waiting 0 ms 2024-11-17T15:37:10,661 DEBUG [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3. 2024-11-17T15:37:10,661 INFO [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(2902): Flushing a80c4e623aab44bea3cfa33ee4fee8f3 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-17T15:37:10,665 DEBUG [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/A/066ab54e34b94717851247eb8e505bd3 is 50, key is test_row_0/A:col10/1731857830473/Put/seqid=0 2024-11-17T15:37:10,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073741933_1109 (size=9857) 2024-11-17T15:37:10,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-17T15:37:11,071 INFO [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=438 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/A/066ab54e34b94717851247eb8e505bd3 2024-11-17T15:37:11,079 DEBUG [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/B/9647091405d34bab98c620eb426e6db7 is 50, key is test_row_0/B:col10/1731857830473/Put/seqid=0 2024-11-17T15:37:11,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073741934_1110 (size=9857) 2024-11-17T15:37:11,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-17T15:37:11,483 INFO [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=438 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/B/9647091405d34bab98c620eb426e6db7 2024-11-17T15:37:11,492 DEBUG [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/C/885e0abfdb394765bf11969f71b47514 is 50, key is test_row_0/C:col10/1731857830473/Put/seqid=0 2024-11-17T15:37:11,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073741935_1111 (size=9857) 2024-11-17T15:37:11,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-17T15:37:11,897 INFO [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=438 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/C/885e0abfdb394765bf11969f71b47514 2024-11-17T15:37:11,903 DEBUG [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/A/066ab54e34b94717851247eb8e505bd3 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/A/066ab54e34b94717851247eb8e505bd3 2024-11-17T15:37:11,907 INFO [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/A/066ab54e34b94717851247eb8e505bd3, entries=100, sequenceid=438, filesize=9.6 K 2024-11-17T15:37:11,908 DEBUG [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/B/9647091405d34bab98c620eb426e6db7 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/B/9647091405d34bab98c620eb426e6db7 2024-11-17T15:37:11,912 INFO [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/B/9647091405d34bab98c620eb426e6db7, entries=100, sequenceid=438, filesize=9.6 K 2024-11-17T15:37:11,913 DEBUG [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/.tmp/C/885e0abfdb394765bf11969f71b47514 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/C/885e0abfdb394765bf11969f71b47514 2024-11-17T15:37:11,917 INFO [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/C/885e0abfdb394765bf11969f71b47514, entries=100, sequenceid=438, filesize=9.6 K 2024-11-17T15:37:11,918 INFO [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for a80c4e623aab44bea3cfa33ee4fee8f3 in 1257ms, sequenceid=438, compaction requested=true 2024-11-17T15:37:11,919 DEBUG [StoreCloser-TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/A/d097e30ef8c34ce6a5c8e59e8f175f5a, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/A/38865204207448788c9dd4870cc1e94b, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/A/19b314941ee246df9bc8cb722efe87f2, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/A/bf4940d3105d4ea09ed03e1888157d51, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/A/f4855d7654e548969ed0272bd95df179, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/A/af2903292537434ab33d6ac72089bda8, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/A/9a30184341e84c8f8a67425ab889451d, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/A/c25a9bb737154ca190744442a7ba939c, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/A/c4d24d5dca594a368262ffbebc6cc41b, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/A/9371bd1061eb4adda17613446d996e31, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/A/f1d805562bc24eacae22cff65e8b3818, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/A/60c5a43971a24642977b31c98205a6d4, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/A/1dc6533af38b4cbe847ca261b396f703, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/A/13ea5809700740d9b5aca2b674dcc378, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/A/a910fc9b9ddd45e6b89cee0071ea5f37, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/A/d73535a23a064d8c926a8b5c93600a45, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/A/b682e790f5a84a3e85029088f19c20aa, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/A/5c7e5227e0aa4248ad3d263081aae7de, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/A/fcbd0df251c94b3ab8d0bba99e7e16c6, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/A/dd4553c4191c4bbb81a5b45b9236645b, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/A/a9d1135dc3a04ef2ad3ec9c701e6aec4, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/A/3429524979594aff8b5088c73a4644fb, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/A/7ecfbd90e83e48deae15512933781ac3, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/A/6593f7ff3eb247f298831d98d93c284d, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/A/3433af5787874e2d87f4b8fb82f5ac5a, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/A/54db723928294532b7790f6c10d7d4bb, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/A/9d5b07bab477452f9c268f8993b0c465, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/A/648fa15c4f8743619ac526b13715f378, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/A/673ac698a90847c2a07f220e762aeae5, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/A/4a5f41439ffe471f94f3f0b7cf34ce80] to archive 2024-11-17T15:37:11,923 DEBUG [StoreCloser-TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-17T15:37:11,928 DEBUG [StoreCloser-TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/A/d097e30ef8c34ce6a5c8e59e8f175f5a to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/A/d097e30ef8c34ce6a5c8e59e8f175f5a 2024-11-17T15:37:11,930 DEBUG [StoreCloser-TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/A/38865204207448788c9dd4870cc1e94b to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/A/38865204207448788c9dd4870cc1e94b 2024-11-17T15:37:11,932 DEBUG [StoreCloser-TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/A/19b314941ee246df9bc8cb722efe87f2 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/A/19b314941ee246df9bc8cb722efe87f2 2024-11-17T15:37:11,933 DEBUG [StoreCloser-TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/A/bf4940d3105d4ea09ed03e1888157d51 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/A/bf4940d3105d4ea09ed03e1888157d51 2024-11-17T15:37:11,935 DEBUG [StoreCloser-TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/A/f4855d7654e548969ed0272bd95df179 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/A/f4855d7654e548969ed0272bd95df179 2024-11-17T15:37:11,936 DEBUG [StoreCloser-TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/A/af2903292537434ab33d6ac72089bda8 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/A/af2903292537434ab33d6ac72089bda8 2024-11-17T15:37:11,938 DEBUG [StoreCloser-TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/A/9a30184341e84c8f8a67425ab889451d to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/A/9a30184341e84c8f8a67425ab889451d 2024-11-17T15:37:11,939 DEBUG [StoreCloser-TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/A/c25a9bb737154ca190744442a7ba939c to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/A/c25a9bb737154ca190744442a7ba939c 2024-11-17T15:37:11,940 DEBUG [StoreCloser-TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/A/c4d24d5dca594a368262ffbebc6cc41b to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/A/c4d24d5dca594a368262ffbebc6cc41b 2024-11-17T15:37:11,941 DEBUG [StoreCloser-TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/A/9371bd1061eb4adda17613446d996e31 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/A/9371bd1061eb4adda17613446d996e31 2024-11-17T15:37:11,943 DEBUG [StoreCloser-TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/A/f1d805562bc24eacae22cff65e8b3818 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/A/f1d805562bc24eacae22cff65e8b3818 2024-11-17T15:37:11,944 DEBUG [StoreCloser-TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/A/60c5a43971a24642977b31c98205a6d4 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/A/60c5a43971a24642977b31c98205a6d4 2024-11-17T15:37:11,945 DEBUG [StoreCloser-TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/A/1dc6533af38b4cbe847ca261b396f703 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/A/1dc6533af38b4cbe847ca261b396f703 2024-11-17T15:37:11,946 DEBUG [StoreCloser-TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/A/13ea5809700740d9b5aca2b674dcc378 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/A/13ea5809700740d9b5aca2b674dcc378 2024-11-17T15:37:11,948 DEBUG [StoreCloser-TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/A/a910fc9b9ddd45e6b89cee0071ea5f37 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/A/a910fc9b9ddd45e6b89cee0071ea5f37 2024-11-17T15:37:11,949 DEBUG [StoreCloser-TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/A/d73535a23a064d8c926a8b5c93600a45 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/A/d73535a23a064d8c926a8b5c93600a45 2024-11-17T15:37:11,951 DEBUG [StoreCloser-TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/A/b682e790f5a84a3e85029088f19c20aa to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/A/b682e790f5a84a3e85029088f19c20aa 2024-11-17T15:37:11,952 DEBUG [StoreCloser-TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/A/5c7e5227e0aa4248ad3d263081aae7de to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/A/5c7e5227e0aa4248ad3d263081aae7de 2024-11-17T15:37:11,954 DEBUG [StoreCloser-TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/A/fcbd0df251c94b3ab8d0bba99e7e16c6 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/A/fcbd0df251c94b3ab8d0bba99e7e16c6 2024-11-17T15:37:11,955 DEBUG [StoreCloser-TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/A/dd4553c4191c4bbb81a5b45b9236645b to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/A/dd4553c4191c4bbb81a5b45b9236645b 2024-11-17T15:37:11,957 DEBUG [StoreCloser-TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/A/a9d1135dc3a04ef2ad3ec9c701e6aec4 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/A/a9d1135dc3a04ef2ad3ec9c701e6aec4 2024-11-17T15:37:11,958 DEBUG [StoreCloser-TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/A/3429524979594aff8b5088c73a4644fb to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/A/3429524979594aff8b5088c73a4644fb 2024-11-17T15:37:11,960 DEBUG [StoreCloser-TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/A/7ecfbd90e83e48deae15512933781ac3 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/A/7ecfbd90e83e48deae15512933781ac3 2024-11-17T15:37:11,961 DEBUG [StoreCloser-TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/A/6593f7ff3eb247f298831d98d93c284d to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/A/6593f7ff3eb247f298831d98d93c284d 2024-11-17T15:37:11,963 DEBUG [StoreCloser-TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/A/3433af5787874e2d87f4b8fb82f5ac5a to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/A/3433af5787874e2d87f4b8fb82f5ac5a 2024-11-17T15:37:11,964 DEBUG [StoreCloser-TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/A/54db723928294532b7790f6c10d7d4bb to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/A/54db723928294532b7790f6c10d7d4bb 2024-11-17T15:37:11,966 DEBUG [StoreCloser-TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/A/9d5b07bab477452f9c268f8993b0c465 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/A/9d5b07bab477452f9c268f8993b0c465 2024-11-17T15:37:11,967 DEBUG [StoreCloser-TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/A/648fa15c4f8743619ac526b13715f378 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/A/648fa15c4f8743619ac526b13715f378 2024-11-17T15:37:11,969 DEBUG [StoreCloser-TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/A/673ac698a90847c2a07f220e762aeae5 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/A/673ac698a90847c2a07f220e762aeae5 2024-11-17T15:37:11,970 DEBUG [StoreCloser-TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/A/4a5f41439ffe471f94f3f0b7cf34ce80 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/A/4a5f41439ffe471f94f3f0b7cf34ce80 2024-11-17T15:37:11,988 DEBUG [StoreCloser-TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/B/73bd1efe107344cfb38a8802ed6648a3, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/B/effa15d312364f1f9fa6183342b6b378, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/B/54f3a827533347dc8644ec6bb313dae9, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/B/a88b22ae672744e884bc4a551099ab62, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/B/173babc1b3494a33bd587d631d8d1e14, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/B/8e3f17193ab349d99e6256a019e0c395, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/B/7dacf2ef7dd947d2ad37db118ff60ac5, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/B/402182b8978946f2a3953e81848a96d0, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/B/cc23f2b2fb9d4a5d813bf9cf1db8de13, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/B/257005ae561f4a7695b8858b5a2dbd6c, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/B/17d92fa5e77244afa19591d96578ff9a, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/B/dcc35d4ce2034fba995cddef4211e265, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/B/edce9a0729d7455dbc263cf9297faef0, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/B/b81c6acf6ced4e1da499caaad6729146, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/B/a057189da82644cb9ca87b340db3879c, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/B/ef6a6b683c8546949102bbbb2136347d, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/B/ffdd0480018b4784b14c6bc32708809f, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/B/41c765f65c0a4c81bcfce4ca50c92d60, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/B/8e27803435684b4d9c7df4ea05b2201d, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/B/7b9e1176d8214ad8b9453f19e186a331, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/B/de3bd57629c149a1af3b41574184fcef, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/B/d8f232cdcdfd4c6c88d6f781bb680cb2, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/B/6671c21f50444f3cbfc0e75479d64a76, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/B/1174f57c6be549f4b24539c1e94ac2c9, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/B/eb4b6c3508384164a4d018fa91b50adf, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/B/9ebb1e22e725441e9be046fdb8ca9246, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/B/2078b0b2034c46fa95fb475c1dcb139d, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/B/654f869346f2457d9fdbf7b3bcf7e61e, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/B/19e2b263a1bf443f824ee0240dc0ccc1, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/B/4f2351093f8840dcb9f93a56fe54793f] to archive 2024-11-17T15:37:11,991 DEBUG [StoreCloser-TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-17T15:37:11,992 DEBUG [StoreCloser-TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/B/73bd1efe107344cfb38a8802ed6648a3 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/B/73bd1efe107344cfb38a8802ed6648a3 2024-11-17T15:37:11,994 DEBUG [StoreCloser-TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/B/effa15d312364f1f9fa6183342b6b378 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/B/effa15d312364f1f9fa6183342b6b378 2024-11-17T15:37:11,995 DEBUG [StoreCloser-TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/B/54f3a827533347dc8644ec6bb313dae9 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/B/54f3a827533347dc8644ec6bb313dae9 2024-11-17T15:37:11,996 DEBUG [StoreCloser-TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/B/a88b22ae672744e884bc4a551099ab62 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/B/a88b22ae672744e884bc4a551099ab62 2024-11-17T15:37:11,998 DEBUG [StoreCloser-TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/B/173babc1b3494a33bd587d631d8d1e14 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/B/173babc1b3494a33bd587d631d8d1e14 2024-11-17T15:37:11,999 DEBUG [StoreCloser-TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/B/8e3f17193ab349d99e6256a019e0c395 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/B/8e3f17193ab349d99e6256a019e0c395 2024-11-17T15:37:12,000 DEBUG [StoreCloser-TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/B/7dacf2ef7dd947d2ad37db118ff60ac5 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/B/7dacf2ef7dd947d2ad37db118ff60ac5 2024-11-17T15:37:12,002 DEBUG [StoreCloser-TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/B/402182b8978946f2a3953e81848a96d0 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/B/402182b8978946f2a3953e81848a96d0 2024-11-17T15:37:12,003 DEBUG [StoreCloser-TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/B/cc23f2b2fb9d4a5d813bf9cf1db8de13 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/B/cc23f2b2fb9d4a5d813bf9cf1db8de13 2024-11-17T15:37:12,004 DEBUG [StoreCloser-TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/B/257005ae561f4a7695b8858b5a2dbd6c to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/B/257005ae561f4a7695b8858b5a2dbd6c 2024-11-17T15:37:12,005 DEBUG [StoreCloser-TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/B/17d92fa5e77244afa19591d96578ff9a to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/B/17d92fa5e77244afa19591d96578ff9a 2024-11-17T15:37:12,006 DEBUG [StoreCloser-TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/B/dcc35d4ce2034fba995cddef4211e265 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/B/dcc35d4ce2034fba995cddef4211e265 2024-11-17T15:37:12,008 DEBUG [StoreCloser-TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/B/edce9a0729d7455dbc263cf9297faef0 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/B/edce9a0729d7455dbc263cf9297faef0 2024-11-17T15:37:12,009 DEBUG [StoreCloser-TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/B/b81c6acf6ced4e1da499caaad6729146 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/B/b81c6acf6ced4e1da499caaad6729146 2024-11-17T15:37:12,010 DEBUG [StoreCloser-TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/B/a057189da82644cb9ca87b340db3879c to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/B/a057189da82644cb9ca87b340db3879c 2024-11-17T15:37:12,012 DEBUG [StoreCloser-TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/B/ef6a6b683c8546949102bbbb2136347d to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/B/ef6a6b683c8546949102bbbb2136347d 2024-11-17T15:37:12,013 DEBUG [StoreCloser-TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/B/ffdd0480018b4784b14c6bc32708809f to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/B/ffdd0480018b4784b14c6bc32708809f 2024-11-17T15:37:12,014 DEBUG [StoreCloser-TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/B/41c765f65c0a4c81bcfce4ca50c92d60 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/B/41c765f65c0a4c81bcfce4ca50c92d60 2024-11-17T15:37:12,016 DEBUG [StoreCloser-TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/B/8e27803435684b4d9c7df4ea05b2201d to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/B/8e27803435684b4d9c7df4ea05b2201d 2024-11-17T15:37:12,017 DEBUG [StoreCloser-TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/B/7b9e1176d8214ad8b9453f19e186a331 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/B/7b9e1176d8214ad8b9453f19e186a331 2024-11-17T15:37:12,018 DEBUG [StoreCloser-TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/B/de3bd57629c149a1af3b41574184fcef to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/B/de3bd57629c149a1af3b41574184fcef 2024-11-17T15:37:12,019 DEBUG [StoreCloser-TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/B/d8f232cdcdfd4c6c88d6f781bb680cb2 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/B/d8f232cdcdfd4c6c88d6f781bb680cb2 2024-11-17T15:37:12,021 DEBUG [StoreCloser-TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/B/6671c21f50444f3cbfc0e75479d64a76 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/B/6671c21f50444f3cbfc0e75479d64a76 2024-11-17T15:37:12,022 DEBUG [StoreCloser-TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/B/1174f57c6be549f4b24539c1e94ac2c9 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/B/1174f57c6be549f4b24539c1e94ac2c9 2024-11-17T15:37:12,023 DEBUG [StoreCloser-TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/B/eb4b6c3508384164a4d018fa91b50adf to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/B/eb4b6c3508384164a4d018fa91b50adf 2024-11-17T15:37:12,024 DEBUG [StoreCloser-TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/B/9ebb1e22e725441e9be046fdb8ca9246 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/B/9ebb1e22e725441e9be046fdb8ca9246 2024-11-17T15:37:12,025 DEBUG [StoreCloser-TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/B/2078b0b2034c46fa95fb475c1dcb139d to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/B/2078b0b2034c46fa95fb475c1dcb139d 2024-11-17T15:37:12,027 DEBUG [StoreCloser-TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/B/654f869346f2457d9fdbf7b3bcf7e61e to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/B/654f869346f2457d9fdbf7b3bcf7e61e 2024-11-17T15:37:12,028 DEBUG [StoreCloser-TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/B/19e2b263a1bf443f824ee0240dc0ccc1 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/B/19e2b263a1bf443f824ee0240dc0ccc1 2024-11-17T15:37:12,029 DEBUG [StoreCloser-TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/B/4f2351093f8840dcb9f93a56fe54793f to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/B/4f2351093f8840dcb9f93a56fe54793f 2024-11-17T15:37:12,030 DEBUG [StoreCloser-TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/C/543e81de5f5d404dbf686651a01cf73f, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/C/460d6067d4d04b5e9ceacf1b12789ddc, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/C/dd28e9fd8ba74381868490f561a28208, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/C/dc66268b3dcc4035a5087570dc0f411a, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/C/ce9ced83164e42da9aa5e6c22eb49366, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/C/75e5d5e3cdb34a5c9816734e84436e6d, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/C/3cfef1ced9b3411fad18d6b66b84dca0, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/C/becd22d7319f4fa8ae277234c1dd5f85, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/C/928d5b7cacba40ad87e24d2af95d5820, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/C/91e5651d57f84b14b08453ad2839a6af, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/C/15a7ee9bbc454ea0ab035b404183d8d0, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/C/70f8dadc523b412eba8acb3b47a2bb3c, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/C/f45426d1a7ca450795c4b3024a9081ea, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/C/8f5ffa1d03ce4f98bbe27cb15aaa48d1, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/C/f2883ee022f14d4aaffca2deacca3e05, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/C/d5914610221d414db8717795a819386c, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/C/fc675b3661dd4d4f9512b303846c4f51, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/C/2f80a378a9834320b80026f1275588d3, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/C/2b647c6640ac4dc0a80050c7dfc5274d, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/C/59133d0cdf4344bfb2b222c734787150, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/C/d6703f4c50504d2986f5f53eab28b6f7, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/C/8c15fb8fc2234ecc981a3a0f8016befb, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/C/0b3e1352b17747328564a6784658e4ae, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/C/53eafeb14b3344349776a7894042175b, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/C/c3ab23d723ab4d028c6e2c788932b11a, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/C/a492ff331e81468f913c445932ccc249, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/C/2d7a58ef8c504d50a868234da5bfb99e, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/C/a670e04ed9b541fea8191a23c27fbfda, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/C/ef8241739fc04978a4a6cd7527470727, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/C/681188ae0d054188b15c451ba61e8e7d] to archive 2024-11-17T15:37:12,031 DEBUG [StoreCloser-TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-17T15:37:12,033 DEBUG [StoreCloser-TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/C/543e81de5f5d404dbf686651a01cf73f to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/C/543e81de5f5d404dbf686651a01cf73f 2024-11-17T15:37:12,034 DEBUG [StoreCloser-TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/C/460d6067d4d04b5e9ceacf1b12789ddc to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/C/460d6067d4d04b5e9ceacf1b12789ddc 2024-11-17T15:37:12,035 DEBUG [StoreCloser-TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/C/dd28e9fd8ba74381868490f561a28208 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/C/dd28e9fd8ba74381868490f561a28208 2024-11-17T15:37:12,036 DEBUG [StoreCloser-TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/C/dc66268b3dcc4035a5087570dc0f411a to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/C/dc66268b3dcc4035a5087570dc0f411a 2024-11-17T15:37:12,038 DEBUG [StoreCloser-TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/C/ce9ced83164e42da9aa5e6c22eb49366 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/C/ce9ced83164e42da9aa5e6c22eb49366 2024-11-17T15:37:12,039 DEBUG [StoreCloser-TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/C/75e5d5e3cdb34a5c9816734e84436e6d to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/C/75e5d5e3cdb34a5c9816734e84436e6d 2024-11-17T15:37:12,041 DEBUG [StoreCloser-TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/C/3cfef1ced9b3411fad18d6b66b84dca0 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/C/3cfef1ced9b3411fad18d6b66b84dca0 2024-11-17T15:37:12,042 DEBUG [StoreCloser-TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/C/becd22d7319f4fa8ae277234c1dd5f85 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/C/becd22d7319f4fa8ae277234c1dd5f85 2024-11-17T15:37:12,044 DEBUG [StoreCloser-TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/C/928d5b7cacba40ad87e24d2af95d5820 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/C/928d5b7cacba40ad87e24d2af95d5820 2024-11-17T15:37:12,046 DEBUG [StoreCloser-TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/C/91e5651d57f84b14b08453ad2839a6af to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/C/91e5651d57f84b14b08453ad2839a6af 2024-11-17T15:37:12,047 DEBUG [StoreCloser-TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/C/15a7ee9bbc454ea0ab035b404183d8d0 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/C/15a7ee9bbc454ea0ab035b404183d8d0 2024-11-17T15:37:12,049 DEBUG [StoreCloser-TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/C/70f8dadc523b412eba8acb3b47a2bb3c to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/C/70f8dadc523b412eba8acb3b47a2bb3c 2024-11-17T15:37:12,050 DEBUG [StoreCloser-TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/C/f45426d1a7ca450795c4b3024a9081ea to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/C/f45426d1a7ca450795c4b3024a9081ea 2024-11-17T15:37:12,052 DEBUG [StoreCloser-TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/C/8f5ffa1d03ce4f98bbe27cb15aaa48d1 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/C/8f5ffa1d03ce4f98bbe27cb15aaa48d1 2024-11-17T15:37:12,053 DEBUG [StoreCloser-TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/C/f2883ee022f14d4aaffca2deacca3e05 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/C/f2883ee022f14d4aaffca2deacca3e05 2024-11-17T15:37:12,055 DEBUG [StoreCloser-TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/C/d5914610221d414db8717795a819386c to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/C/d5914610221d414db8717795a819386c 2024-11-17T15:37:12,056 DEBUG [StoreCloser-TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/C/fc675b3661dd4d4f9512b303846c4f51 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/C/fc675b3661dd4d4f9512b303846c4f51 2024-11-17T15:37:12,057 DEBUG [StoreCloser-TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/C/2f80a378a9834320b80026f1275588d3 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/C/2f80a378a9834320b80026f1275588d3 2024-11-17T15:37:12,059 DEBUG [StoreCloser-TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/C/2b647c6640ac4dc0a80050c7dfc5274d to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/C/2b647c6640ac4dc0a80050c7dfc5274d 2024-11-17T15:37:12,060 DEBUG [StoreCloser-TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/C/59133d0cdf4344bfb2b222c734787150 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/C/59133d0cdf4344bfb2b222c734787150 2024-11-17T15:37:12,062 DEBUG [StoreCloser-TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/C/d6703f4c50504d2986f5f53eab28b6f7 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/C/d6703f4c50504d2986f5f53eab28b6f7 2024-11-17T15:37:12,063 DEBUG [StoreCloser-TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/C/8c15fb8fc2234ecc981a3a0f8016befb to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/C/8c15fb8fc2234ecc981a3a0f8016befb 2024-11-17T15:37:12,064 DEBUG [StoreCloser-TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/C/0b3e1352b17747328564a6784658e4ae to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/C/0b3e1352b17747328564a6784658e4ae 2024-11-17T15:37:12,066 DEBUG [StoreCloser-TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/C/53eafeb14b3344349776a7894042175b to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/C/53eafeb14b3344349776a7894042175b 2024-11-17T15:37:12,067 DEBUG [StoreCloser-TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/C/c3ab23d723ab4d028c6e2c788932b11a to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/C/c3ab23d723ab4d028c6e2c788932b11a 2024-11-17T15:37:12,068 DEBUG [StoreCloser-TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/C/a492ff331e81468f913c445932ccc249 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/C/a492ff331e81468f913c445932ccc249 2024-11-17T15:37:12,069 DEBUG [StoreCloser-TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/C/2d7a58ef8c504d50a868234da5bfb99e to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/C/2d7a58ef8c504d50a868234da5bfb99e 2024-11-17T15:37:12,071 DEBUG [StoreCloser-TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/C/a670e04ed9b541fea8191a23c27fbfda to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/C/a670e04ed9b541fea8191a23c27fbfda 2024-11-17T15:37:12,072 DEBUG [StoreCloser-TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/C/ef8241739fc04978a4a6cd7527470727 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/C/ef8241739fc04978a4a6cd7527470727 2024-11-17T15:37:12,073 DEBUG [StoreCloser-TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/C/681188ae0d054188b15c451ba61e8e7d to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/C/681188ae0d054188b15c451ba61e8e7d 2024-11-17T15:37:12,078 DEBUG [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/recovered.edits/441.seqid, newMaxSeqId=441, maxSeqId=1 2024-11-17T15:37:12,079 INFO [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3. 2024-11-17T15:37:12,079 DEBUG [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1676): Region close journal for a80c4e623aab44bea3cfa33ee4fee8f3: Waiting for close lock at 1731857830660Running coprocessor pre-close hooks at 1731857830660Disabling compacts and flushes for region at 1731857830660Disabling writes for close at 1731857830661 (+1 ms)Obtaining lock to block concurrent updates at 1731857830661Preparing flush snapshotting stores in a80c4e623aab44bea3cfa33ee4fee8f3 at 1731857830661Finished memstore snapshotting TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1731857830661Flushing stores of TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3. at 1731857830662 (+1 ms)Flushing a80c4e623aab44bea3cfa33ee4fee8f3/A: creating writer at 1731857830662Flushing a80c4e623aab44bea3cfa33ee4fee8f3/A: appending metadata at 1731857830665 (+3 ms)Flushing a80c4e623aab44bea3cfa33ee4fee8f3/A: closing flushed file at 1731857830665Flushing a80c4e623aab44bea3cfa33ee4fee8f3/B: creating writer at 1731857831076 (+411 ms)Flushing a80c4e623aab44bea3cfa33ee4fee8f3/B: appending metadata at 1731857831078 (+2 ms)Flushing a80c4e623aab44bea3cfa33ee4fee8f3/B: closing flushed file at 1731857831078Flushing a80c4e623aab44bea3cfa33ee4fee8f3/C: creating writer at 1731857831488 (+410 ms)Flushing a80c4e623aab44bea3cfa33ee4fee8f3/C: appending metadata at 1731857831491 (+3 ms)Flushing a80c4e623aab44bea3cfa33ee4fee8f3/C: closing flushed file at 1731857831491Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@595dd370: reopening flushed file at 1731857831902 (+411 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@24095a26: reopening flushed file at 1731857831907 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@71935530: reopening flushed file at 1731857831913 (+6 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for a80c4e623aab44bea3cfa33ee4fee8f3 in 1257ms, sequenceid=438, compaction requested=true at 1731857831918 (+5 ms)Writing region close event to WAL at 1731857832074 (+156 ms)Running coprocessor post-close hooks at 1731857832079 (+5 ms)Closed at 1731857832079 2024-11-17T15:37:12,082 INFO [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(157): Closed a80c4e623aab44bea3cfa33ee4fee8f3 2024-11-17T15:37:12,082 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=a80c4e623aab44bea3cfa33ee4fee8f3, regionState=CLOSED 2024-11-17T15:37:12,084 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=29, state=RUNNABLE, hasLock=false; CloseRegionProcedure a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 because future has completed 2024-11-17T15:37:12,086 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-11-17T15:37:12,087 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; CloseRegionProcedure a80c4e623aab44bea3cfa33ee4fee8f3, server=7a780d55532c,32955,1731857800648 in 1.5820 sec 2024-11-17T15:37:12,088 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=29, resume processing ppid=28 2024-11-17T15:37:12,088 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=28, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a80c4e623aab44bea3cfa33ee4fee8f3, UNASSIGN in 1.5890 sec 2024-11-17T15:37:12,091 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-11-17T15:37:12,092 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5970 sec 2024-11-17T15:37:12,093 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731857832093"}]},"ts":"1731857832093"} 2024-11-17T15:37:12,094 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-17T15:37:12,094 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-17T15:37:12,096 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.6110 sec 2024-11-17T15:37:12,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-17T15:37:12,633 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-17T15:37:12,637 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-17T15:37:12,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] procedure2.ProcedureExecutor(1139): Stored pid=31, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T15:37:12,642 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=31, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T15:37:12,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-17T15:37:12,644 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=31, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T15:37:12,649 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3 2024-11-17T15:37:12,653 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/A, FileablePath, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/B, FileablePath, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/C, FileablePath, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/recovered.edits] 2024-11-17T15:37:12,657 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/A/066ab54e34b94717851247eb8e505bd3 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/A/066ab54e34b94717851247eb8e505bd3 2024-11-17T15:37:12,658 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/A/13a92a9f13f844ef944ddff7f62d806a to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/A/13a92a9f13f844ef944ddff7f62d806a 2024-11-17T15:37:12,660 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/A/4e7c6202aab5454ba2c2b30341ee8416 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/A/4e7c6202aab5454ba2c2b30341ee8416 2024-11-17T15:37:12,663 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/B/0cbd9ce0277c4186bcc7410decd2118e to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/B/0cbd9ce0277c4186bcc7410decd2118e 2024-11-17T15:37:12,664 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/B/42742a62c6a24f109f1bde0fabc13d69 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/B/42742a62c6a24f109f1bde0fabc13d69 2024-11-17T15:37:12,665 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/B/9647091405d34bab98c620eb426e6db7 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/B/9647091405d34bab98c620eb426e6db7 2024-11-17T15:37:12,668 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/C/4740d0bfb38942809bc7cc5c26a3a155 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/C/4740d0bfb38942809bc7cc5c26a3a155 2024-11-17T15:37:12,669 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/C/7420f113336f4966a5e53be569edce53 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/C/7420f113336f4966a5e53be569edce53 2024-11-17T15:37:12,670 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/C/885e0abfdb394765bf11969f71b47514 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/C/885e0abfdb394765bf11969f71b47514 2024-11-17T15:37:12,674 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/recovered.edits/441.seqid to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3/recovered.edits/441.seqid 2024-11-17T15:37:12,675 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a80c4e623aab44bea3cfa33ee4fee8f3 2024-11-17T15:37:12,675 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-17T15:37:12,680 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=31, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T15:37:12,684 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32955 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-11-17T15:37:12,689 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-17T15:37:12,723 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-17T15:37:12,724 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=31, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T15:37:12,724 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-17T15:37:12,725 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731857832724"}]},"ts":"9223372036854775807"} 2024-11-17T15:37:12,729 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-17T15:37:12,729 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => a80c4e623aab44bea3cfa33ee4fee8f3, NAME => 'TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3.', STARTKEY => '', ENDKEY => ''}] 2024-11-17T15:37:12,730 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-17T15:37:12,730 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731857832730"}]},"ts":"9223372036854775807"} 2024-11-17T15:37:12,733 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-17T15:37:12,734 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=31, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T15:37:12,735 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 97 msec 2024-11-17T15:37:12,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-17T15:37:12,754 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-17T15:37:12,754 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-17T15:37:12,769 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMixedAtomicity Thread=238 (was 211) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/7a780d55532c:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1842016799_22 at /127.0.0.1:39738 [Waiting for operation #314] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1410560447_22 at /127.0.0.1:39884 [Waiting for operation #304] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;7a780d55532c:32955-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=453 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=437 (was 223) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=3341 (was 3925) 2024-11-17T15:37:12,781 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobMixedAtomicity Thread=238, OpenFileDescriptor=453, MaxFileDescriptor=1048576, SystemLoadAverage=437, ProcessCount=11, AvailableMemoryMB=3341 2024-11-17T15:37:12,783 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-17T15:37:12,784 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-17T15:37:12,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] procedure2.ProcedureExecutor(1139): Stored pid=32, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-17T15:37:12,786 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-17T15:37:12,786 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:12,786 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 32 2024-11-17T15:37:12,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-17T15:37:12,787 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-17T15:37:12,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073741936_1112 (size=959) 2024-11-17T15:37:12,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-17T15:37:13,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-17T15:37:13,196 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 34c72dd45fe3d9e6a1558a16ef53eb06, NAME => 'TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206 2024-11-17T15:37:13,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073741937_1113 (size=53) 2024-11-17T15:37:13,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-17T15:37:13,604 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T15:37:13,604 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 34c72dd45fe3d9e6a1558a16ef53eb06, disabling compactions & flushes 2024-11-17T15:37:13,604 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. 2024-11-17T15:37:13,604 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. 2024-11-17T15:37:13,604 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. after waiting 0 ms 2024-11-17T15:37:13,604 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. 2024-11-17T15:37:13,604 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. 2024-11-17T15:37:13,604 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 34c72dd45fe3d9e6a1558a16ef53eb06: Waiting for close lock at 1731857833604Disabling compacts and flushes for region at 1731857833604Disabling writes for close at 1731857833604Writing region close event to WAL at 1731857833604Closed at 1731857833604 2024-11-17T15:37:13,606 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-17T15:37:13,606 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731857833606"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731857833606"}]},"ts":"1731857833606"} 2024-11-17T15:37:13,608 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-17T15:37:13,609 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-17T15:37:13,609 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731857833609"}]},"ts":"1731857833609"} 2024-11-17T15:37:13,611 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-17T15:37:13,611 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=34c72dd45fe3d9e6a1558a16ef53eb06, ASSIGN}] 2024-11-17T15:37:13,612 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=34c72dd45fe3d9e6a1558a16ef53eb06, ASSIGN 2024-11-17T15:37:13,613 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=34c72dd45fe3d9e6a1558a16ef53eb06, ASSIGN; state=OFFLINE, location=7a780d55532c,32955,1731857800648; forceNewPlan=false, retain=false 2024-11-17T15:37:13,763 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=34c72dd45fe3d9e6a1558a16ef53eb06, regionState=OPENING, regionLocation=7a780d55532c,32955,1731857800648 2024-11-17T15:37:13,765 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=34c72dd45fe3d9e6a1558a16ef53eb06, ASSIGN because future has completed 2024-11-17T15:37:13,766 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=33, state=RUNNABLE, hasLock=false; OpenRegionProcedure 34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648}] 2024-11-17T15:37:13,921 INFO [RS_OPEN_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. 2024-11-17T15:37:13,921 DEBUG [RS_OPEN_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7752): Opening region: {ENCODED => 34c72dd45fe3d9e6a1558a16ef53eb06, NAME => 'TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.', STARTKEY => '', ENDKEY => ''} 2024-11-17T15:37:13,921 DEBUG [RS_OPEN_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 34c72dd45fe3d9e6a1558a16ef53eb06 2024-11-17T15:37:13,921 DEBUG [RS_OPEN_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T15:37:13,922 DEBUG [RS_OPEN_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7794): checking encryption for 34c72dd45fe3d9e6a1558a16ef53eb06 2024-11-17T15:37:13,922 DEBUG [RS_OPEN_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7797): checking classloading for 34c72dd45fe3d9e6a1558a16ef53eb06 2024-11-17T15:37:13,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-17T15:37:13,923 INFO [StoreOpener-34c72dd45fe3d9e6a1558a16ef53eb06-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 34c72dd45fe3d9e6a1558a16ef53eb06 2024-11-17T15:37:13,925 INFO [StoreOpener-34c72dd45fe3d9e6a1558a16ef53eb06-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 34c72dd45fe3d9e6a1558a16ef53eb06 columnFamilyName A 2024-11-17T15:37:13,925 DEBUG [StoreOpener-34c72dd45fe3d9e6a1558a16ef53eb06-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:13,926 INFO [StoreOpener-34c72dd45fe3d9e6a1558a16ef53eb06-1 {}] regionserver.HStore(327): Store=34c72dd45fe3d9e6a1558a16ef53eb06/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T15:37:13,926 INFO [StoreOpener-34c72dd45fe3d9e6a1558a16ef53eb06-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 34c72dd45fe3d9e6a1558a16ef53eb06 2024-11-17T15:37:13,927 INFO [StoreOpener-34c72dd45fe3d9e6a1558a16ef53eb06-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 34c72dd45fe3d9e6a1558a16ef53eb06 columnFamilyName B 2024-11-17T15:37:13,927 DEBUG [StoreOpener-34c72dd45fe3d9e6a1558a16ef53eb06-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:13,928 INFO [StoreOpener-34c72dd45fe3d9e6a1558a16ef53eb06-1 {}] regionserver.HStore(327): Store=34c72dd45fe3d9e6a1558a16ef53eb06/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T15:37:13,928 INFO [StoreOpener-34c72dd45fe3d9e6a1558a16ef53eb06-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 34c72dd45fe3d9e6a1558a16ef53eb06 2024-11-17T15:37:13,929 INFO [StoreOpener-34c72dd45fe3d9e6a1558a16ef53eb06-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 34c72dd45fe3d9e6a1558a16ef53eb06 columnFamilyName C 2024-11-17T15:37:13,929 DEBUG [StoreOpener-34c72dd45fe3d9e6a1558a16ef53eb06-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:13,930 INFO [StoreOpener-34c72dd45fe3d9e6a1558a16ef53eb06-1 {}] regionserver.HStore(327): Store=34c72dd45fe3d9e6a1558a16ef53eb06/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T15:37:13,930 DEBUG [RS_OPEN_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1038): replaying wal for 34c72dd45fe3d9e6a1558a16ef53eb06 2024-11-17T15:37:13,930 DEBUG [RS_OPEN_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06 2024-11-17T15:37:13,931 DEBUG [RS_OPEN_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06 2024-11-17T15:37:13,932 DEBUG [RS_OPEN_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1048): stopping wal replay for 34c72dd45fe3d9e6a1558a16ef53eb06 2024-11-17T15:37:13,932 DEBUG [RS_OPEN_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1060): Cleaning up temporary data for 34c72dd45fe3d9e6a1558a16ef53eb06 2024-11-17T15:37:13,932 DEBUG [RS_OPEN_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-17T15:37:13,933 DEBUG [RS_OPEN_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1093): writing seq id for 34c72dd45fe3d9e6a1558a16ef53eb06 2024-11-17T15:37:13,935 DEBUG [RS_OPEN_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-17T15:37:13,936 INFO [RS_OPEN_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1114): Opened 34c72dd45fe3d9e6a1558a16ef53eb06; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65632618, jitterRate=-0.021997779607772827}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-17T15:37:13,936 DEBUG [RS_OPEN_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 34c72dd45fe3d9e6a1558a16ef53eb06 2024-11-17T15:37:13,937 DEBUG [RS_OPEN_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1006): Region open journal for 34c72dd45fe3d9e6a1558a16ef53eb06: Running coprocessor pre-open hook at 1731857833922Writing region info on filesystem at 1731857833922Initializing all the Stores at 1731857833922Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731857833922Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731857833923 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731857833923Cleaning up temporary data from old regions at 1731857833932 (+9 ms)Running coprocessor post-open hooks at 1731857833936 (+4 ms)Region opened successfully at 1731857833937 (+1 ms) 2024-11-17T15:37:13,938 INFO [RS_OPEN_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., pid=34, masterSystemTime=1731857833917 2024-11-17T15:37:13,940 DEBUG [RS_OPEN_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. 2024-11-17T15:37:13,940 INFO [RS_OPEN_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. 2024-11-17T15:37:13,940 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=34c72dd45fe3d9e6a1558a16ef53eb06, regionState=OPEN, openSeqNum=2, regionLocation=7a780d55532c,32955,1731857800648 2024-11-17T15:37:13,942 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=33, state=RUNNABLE, hasLock=false; OpenRegionProcedure 34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 because future has completed 2024-11-17T15:37:13,945 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=33 2024-11-17T15:37:13,945 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=33, state=SUCCESS, hasLock=false; OpenRegionProcedure 34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 in 177 msec 2024-11-17T15:37:13,947 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=32 2024-11-17T15:37:13,947 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=34c72dd45fe3d9e6a1558a16ef53eb06, ASSIGN in 334 msec 2024-11-17T15:37:13,948 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-17T15:37:13,948 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731857833948"}]},"ts":"1731857833948"} 2024-11-17T15:37:13,950 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-17T15:37:13,951 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-17T15:37:13,953 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1670 sec 2024-11-17T15:37:14,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-17T15:37:14,933 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-17T15:37:14,934 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ffee9aa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:37:14,934 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7a780d55532c,44223,-1 for getting cluster id 2024-11-17T15:37:14,935 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T15:37:14,936 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e01f4cd2-8971-4a73-861a-2ca0deb4f76e' 2024-11-17T15:37:14,936 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T15:37:14,936 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e01f4cd2-8971-4a73-861a-2ca0deb4f76e" 2024-11-17T15:37:14,937 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29de59cc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:37:14,937 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7a780d55532c,44223,-1] 2024-11-17T15:37:14,937 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T15:37:14,937 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:37:14,938 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38016, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T15:37:14,939 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1953facc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:37:14,939 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T15:37:14,940 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7a780d55532c,32955,1731857800648, seqNum=-1] 2024-11-17T15:37:14,941 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T15:37:14,942 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48350, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T15:37:14,944 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-17T15:37:14,945 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is 7a780d55532c,44223,1731857799978 2024-11-17T15:37:14,945 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@12dd14a7 2024-11-17T15:37:14,945 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-17T15:37:14,946 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38032, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-17T15:37:14,951 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-17T15:37:14,951 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-17T15:37:14,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] procedure2.ProcedureExecutor(1139): Stored pid=35, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-17T15:37:14,968 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073741938_1114 (size=995) 2024-11-17T15:37:15,370 DEBUG [PEWorker-2 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-17T15:37:15,370 INFO [PEWorker-2 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-17T15:37:15,374 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=35, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-17T15:37:15,380 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=34c72dd45fe3d9e6a1558a16ef53eb06, REOPEN/MOVE}] 2024-11-17T15:37:15,381 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=34c72dd45fe3d9e6a1558a16ef53eb06, REOPEN/MOVE 2024-11-17T15:37:15,382 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=34c72dd45fe3d9e6a1558a16ef53eb06, regionState=CLOSING, regionLocation=7a780d55532c,32955,1731857800648 2024-11-17T15:37:15,383 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=34c72dd45fe3d9e6a1558a16ef53eb06, REOPEN/MOVE because future has completed 2024-11-17T15:37:15,384 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-17T15:37:15,384 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=37, state=RUNNABLE, hasLock=false; CloseRegionProcedure 34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648}] 2024-11-17T15:37:15,537 INFO [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] handler.UnassignRegionHandler(122): Close 34c72dd45fe3d9e6a1558a16ef53eb06 2024-11-17T15:37:15,537 DEBUG [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-17T15:37:15,537 DEBUG [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1722): Closing 34c72dd45fe3d9e6a1558a16ef53eb06, disabling compactions & flushes 2024-11-17T15:37:15,537 INFO [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. 2024-11-17T15:37:15,537 DEBUG [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. 2024-11-17T15:37:15,537 DEBUG [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. after waiting 0 ms 2024-11-17T15:37:15,537 DEBUG [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. 2024-11-17T15:37:15,541 DEBUG [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-17T15:37:15,542 INFO [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. 2024-11-17T15:37:15,542 DEBUG [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1676): Region close journal for 34c72dd45fe3d9e6a1558a16ef53eb06: Waiting for close lock at 1731857835537Running coprocessor pre-close hooks at 1731857835537Disabling compacts and flushes for region at 1731857835537Disabling writes for close at 1731857835537Writing region close event to WAL at 1731857835538 (+1 ms)Running coprocessor post-close hooks at 1731857835541 (+3 ms)Closed at 1731857835542 (+1 ms) 2024-11-17T15:37:15,542 WARN [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegionServer(3299): Not adding moved region record: 34c72dd45fe3d9e6a1558a16ef53eb06 to self. 2024-11-17T15:37:15,544 INFO [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] handler.UnassignRegionHandler(157): Closed 34c72dd45fe3d9e6a1558a16ef53eb06 2024-11-17T15:37:15,544 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=34c72dd45fe3d9e6a1558a16ef53eb06, regionState=CLOSED 2024-11-17T15:37:15,546 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=38, ppid=37, state=RUNNABLE, hasLock=false; CloseRegionProcedure 34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 because future has completed 2024-11-17T15:37:15,548 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=38, resume processing ppid=37 2024-11-17T15:37:15,548 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, ppid=37, state=SUCCESS, hasLock=false; CloseRegionProcedure 34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 in 163 msec 2024-11-17T15:37:15,549 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=34c72dd45fe3d9e6a1558a16ef53eb06, REOPEN/MOVE; state=CLOSED, location=7a780d55532c,32955,1731857800648; forceNewPlan=false, retain=true 2024-11-17T15:37:15,700 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=34c72dd45fe3d9e6a1558a16ef53eb06, regionState=OPENING, regionLocation=7a780d55532c,32955,1731857800648 2024-11-17T15:37:15,701 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=34c72dd45fe3d9e6a1558a16ef53eb06, REOPEN/MOVE because future has completed 2024-11-17T15:37:15,702 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=37, state=RUNNABLE, hasLock=false; OpenRegionProcedure 34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648}] 2024-11-17T15:37:15,857 INFO [RS_OPEN_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. 2024-11-17T15:37:15,857 DEBUG [RS_OPEN_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7752): Opening region: {ENCODED => 34c72dd45fe3d9e6a1558a16ef53eb06, NAME => 'TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.', STARTKEY => '', ENDKEY => ''} 2024-11-17T15:37:15,857 DEBUG [RS_OPEN_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 34c72dd45fe3d9e6a1558a16ef53eb06 2024-11-17T15:37:15,858 DEBUG [RS_OPEN_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T15:37:15,858 DEBUG [RS_OPEN_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7794): checking encryption for 34c72dd45fe3d9e6a1558a16ef53eb06 2024-11-17T15:37:15,858 DEBUG [RS_OPEN_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7797): checking classloading for 34c72dd45fe3d9e6a1558a16ef53eb06 2024-11-17T15:37:15,860 INFO [StoreOpener-34c72dd45fe3d9e6a1558a16ef53eb06-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 34c72dd45fe3d9e6a1558a16ef53eb06 2024-11-17T15:37:15,865 INFO [StoreOpener-34c72dd45fe3d9e6a1558a16ef53eb06-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 34c72dd45fe3d9e6a1558a16ef53eb06 columnFamilyName A 2024-11-17T15:37:15,867 DEBUG [StoreOpener-34c72dd45fe3d9e6a1558a16ef53eb06-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:15,868 INFO [StoreOpener-34c72dd45fe3d9e6a1558a16ef53eb06-1 {}] regionserver.HStore(327): Store=34c72dd45fe3d9e6a1558a16ef53eb06/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T15:37:15,869 INFO [StoreOpener-34c72dd45fe3d9e6a1558a16ef53eb06-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 34c72dd45fe3d9e6a1558a16ef53eb06 2024-11-17T15:37:15,869 INFO [StoreOpener-34c72dd45fe3d9e6a1558a16ef53eb06-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 34c72dd45fe3d9e6a1558a16ef53eb06 columnFamilyName B 2024-11-17T15:37:15,870 DEBUG [StoreOpener-34c72dd45fe3d9e6a1558a16ef53eb06-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:15,870 INFO [StoreOpener-34c72dd45fe3d9e6a1558a16ef53eb06-1 {}] regionserver.HStore(327): Store=34c72dd45fe3d9e6a1558a16ef53eb06/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T15:37:15,870 INFO [StoreOpener-34c72dd45fe3d9e6a1558a16ef53eb06-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 34c72dd45fe3d9e6a1558a16ef53eb06 2024-11-17T15:37:15,871 INFO [StoreOpener-34c72dd45fe3d9e6a1558a16ef53eb06-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 34c72dd45fe3d9e6a1558a16ef53eb06 columnFamilyName C 2024-11-17T15:37:15,871 DEBUG [StoreOpener-34c72dd45fe3d9e6a1558a16ef53eb06-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:15,871 INFO [StoreOpener-34c72dd45fe3d9e6a1558a16ef53eb06-1 {}] regionserver.HStore(327): Store=34c72dd45fe3d9e6a1558a16ef53eb06/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T15:37:15,872 DEBUG [RS_OPEN_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1038): replaying wal for 34c72dd45fe3d9e6a1558a16ef53eb06 2024-11-17T15:37:15,872 DEBUG [RS_OPEN_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06 2024-11-17T15:37:15,873 DEBUG [RS_OPEN_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06 2024-11-17T15:37:15,874 DEBUG [RS_OPEN_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1048): stopping wal replay for 34c72dd45fe3d9e6a1558a16ef53eb06 2024-11-17T15:37:15,874 DEBUG [RS_OPEN_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1060): Cleaning up temporary data for 34c72dd45fe3d9e6a1558a16ef53eb06 2024-11-17T15:37:15,875 DEBUG [RS_OPEN_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-17T15:37:15,876 DEBUG [RS_OPEN_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1093): writing seq id for 34c72dd45fe3d9e6a1558a16ef53eb06 2024-11-17T15:37:15,877 INFO [RS_OPEN_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1114): Opened 34c72dd45fe3d9e6a1558a16ef53eb06; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61815517, jitterRate=-0.07887701690196991}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-17T15:37:15,877 DEBUG [RS_OPEN_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 34c72dd45fe3d9e6a1558a16ef53eb06 2024-11-17T15:37:15,878 DEBUG [RS_OPEN_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1006): Region open journal for 34c72dd45fe3d9e6a1558a16ef53eb06: Running coprocessor pre-open hook at 1731857835858Writing region info on filesystem at 1731857835858Initializing all the Stores at 1731857835859 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731857835859Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731857835859Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731857835859Cleaning up temporary data from old regions at 1731857835874 (+15 ms)Running coprocessor post-open hooks at 1731857835877 (+3 ms)Region opened successfully at 1731857835878 (+1 ms) 2024-11-17T15:37:15,879 INFO [RS_OPEN_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., pid=39, masterSystemTime=1731857835853 2024-11-17T15:37:15,881 DEBUG [RS_OPEN_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. 2024-11-17T15:37:15,881 INFO [RS_OPEN_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. 2024-11-17T15:37:15,881 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=34c72dd45fe3d9e6a1558a16ef53eb06, regionState=OPEN, openSeqNum=5, regionLocation=7a780d55532c,32955,1731857800648 2024-11-17T15:37:15,883 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=37, state=RUNNABLE, hasLock=false; OpenRegionProcedure 34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 because future has completed 2024-11-17T15:37:15,886 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=37 2024-11-17T15:37:15,886 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=37, state=SUCCESS, hasLock=false; OpenRegionProcedure 34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 in 183 msec 2024-11-17T15:37:15,888 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=36 2024-11-17T15:37:15,888 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=36, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=34c72dd45fe3d9e6a1558a16ef53eb06, REOPEN/MOVE in 506 msec 2024-11-17T15:37:15,891 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=35 2024-11-17T15:37:15,891 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=35, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 516 msec 2024-11-17T15:37:15,895 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 939 msec 2024-11-17T15:37:15,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=35 2024-11-17T15:37:15,898 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-17T15:37:15,898 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f683619, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:37:15,898 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7a780d55532c,44223,-1 for getting cluster id 2024-11-17T15:37:15,898 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T15:37:15,899 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e01f4cd2-8971-4a73-861a-2ca0deb4f76e' 2024-11-17T15:37:15,900 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T15:37:15,900 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e01f4cd2-8971-4a73-861a-2ca0deb4f76e" 2024-11-17T15:37:15,900 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22b08738, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:37:15,900 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7a780d55532c,44223,-1] 2024-11-17T15:37:15,900 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T15:37:15,900 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:37:15,901 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38042, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T15:37:15,902 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@36aef75, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:37:15,902 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@dd47c94, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:37:15,902 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7a780d55532c,44223,-1 for getting cluster id 2024-11-17T15:37:15,903 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T15:37:15,903 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e01f4cd2-8971-4a73-861a-2ca0deb4f76e' 2024-11-17T15:37:15,903 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T15:37:15,904 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e01f4cd2-8971-4a73-861a-2ca0deb4f76e" 2024-11-17T15:37:15,904 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46ea57d9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:37:15,904 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7a780d55532c,44223,-1] 2024-11-17T15:37:15,904 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T15:37:15,904 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:37:15,905 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38062, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T15:37:15,906 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ade52af, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:37:15,906 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@750c1845, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:37:15,906 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7a780d55532c,44223,-1 for getting cluster id 2024-11-17T15:37:15,907 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T15:37:15,907 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e01f4cd2-8971-4a73-861a-2ca0deb4f76e' 2024-11-17T15:37:15,907 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T15:37:15,907 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e01f4cd2-8971-4a73-861a-2ca0deb4f76e" 2024-11-17T15:37:15,908 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4939cbd9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:37:15,908 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7a780d55532c,44223,-1] 2024-11-17T15:37:15,908 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T15:37:15,908 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:37:15,909 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38090, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T15:37:15,909 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50c61e5b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:37:15,910 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@85a5f7a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:37:15,910 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7a780d55532c,44223,-1 for getting cluster id 2024-11-17T15:37:15,910 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T15:37:15,911 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e01f4cd2-8971-4a73-861a-2ca0deb4f76e' 2024-11-17T15:37:15,911 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T15:37:15,911 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e01f4cd2-8971-4a73-861a-2ca0deb4f76e" 2024-11-17T15:37:15,911 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c1932d1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:37:15,911 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7a780d55532c,44223,-1] 2024-11-17T15:37:15,911 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T15:37:15,912 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:37:15,912 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38104, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T15:37:15,913 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31d48104, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:37:15,914 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71151d6d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:37:15,914 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7a780d55532c,44223,-1 for getting cluster id 2024-11-17T15:37:15,914 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T15:37:15,914 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e01f4cd2-8971-4a73-861a-2ca0deb4f76e' 2024-11-17T15:37:15,915 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T15:37:15,915 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e01f4cd2-8971-4a73-861a-2ca0deb4f76e" 2024-11-17T15:37:15,915 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@631ec949, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:37:15,915 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7a780d55532c,44223,-1] 2024-11-17T15:37:15,915 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T15:37:15,915 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:37:15,916 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38122, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T15:37:15,917 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72164a5c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:37:15,918 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ab1fb6e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:37:15,918 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7a780d55532c,44223,-1 for getting cluster id 2024-11-17T15:37:15,918 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T15:37:15,918 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e01f4cd2-8971-4a73-861a-2ca0deb4f76e' 2024-11-17T15:37:15,918 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T15:37:15,918 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e01f4cd2-8971-4a73-861a-2ca0deb4f76e" 2024-11-17T15:37:15,919 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43b597e9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:37:15,919 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7a780d55532c,44223,-1] 2024-11-17T15:37:15,919 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T15:37:15,919 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:37:15,920 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38136, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T15:37:15,920 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4de3f8b0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:37:15,921 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6dd18334, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:37:15,921 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7a780d55532c,44223,-1 for getting cluster id 2024-11-17T15:37:15,921 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T15:37:15,922 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e01f4cd2-8971-4a73-861a-2ca0deb4f76e' 2024-11-17T15:37:15,922 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T15:37:15,922 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e01f4cd2-8971-4a73-861a-2ca0deb4f76e" 2024-11-17T15:37:15,922 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@757c90f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:37:15,922 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7a780d55532c,44223,-1] 2024-11-17T15:37:15,922 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T15:37:15,923 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:37:15,923 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38154, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T15:37:15,924 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d024d89, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:37:15,926 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52c57d01, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:37:15,926 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7a780d55532c,44223,-1 for getting cluster id 2024-11-17T15:37:15,926 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T15:37:15,927 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e01f4cd2-8971-4a73-861a-2ca0deb4f76e' 2024-11-17T15:37:15,927 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T15:37:15,927 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e01f4cd2-8971-4a73-861a-2ca0deb4f76e" 2024-11-17T15:37:15,927 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f19a477, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:37:15,927 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7a780d55532c,44223,-1] 2024-11-17T15:37:15,928 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T15:37:15,928 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:37:15,929 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38170, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T15:37:15,929 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@17549e9a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:37:15,930 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b465b92, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:37:15,930 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7a780d55532c,44223,-1 for getting cluster id 2024-11-17T15:37:15,930 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T15:37:15,931 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e01f4cd2-8971-4a73-861a-2ca0deb4f76e' 2024-11-17T15:37:15,931 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T15:37:15,931 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e01f4cd2-8971-4a73-861a-2ca0deb4f76e" 2024-11-17T15:37:15,931 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@596890, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:37:15,931 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7a780d55532c,44223,-1] 2024-11-17T15:37:15,931 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T15:37:15,931 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:37:15,932 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38198, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T15:37:15,933 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3fb7526, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:37:15,933 DEBUG [Thread-597 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T15:37:15,933 DEBUG [Thread-590 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T15:37:15,933 DEBUG [Thread-603 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T15:37:15,934 DEBUG [Thread-599 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T15:37:15,934 DEBUG [Thread-601 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T15:37:15,934 DEBUG [Thread-588 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T15:37:15,934 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T15:37:15,935 DEBUG [Thread-594 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T15:37:15,935 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7a780d55532c,32955,1731857800648, seqNum=-1] 2024-11-17T15:37:15,935 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7a780d55532c,32955,1731857800648, seqNum=-1] 2024-11-17T15:37:15,935 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7a780d55532c,32955,1731857800648, seqNum=-1] 2024-11-17T15:37:15,935 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T15:37:15,935 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7a780d55532c,32955,1731857800648, seqNum=-1] 2024-11-17T15:37:15,935 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7a780d55532c,32955,1731857800648, seqNum=-1] 2024-11-17T15:37:15,936 DEBUG [Thread-588 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7a780d55532c,32955,1731857800648, seqNum=-1] 2024-11-17T15:37:15,936 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T15:37:15,936 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T15:37:15,936 DEBUG [Thread-588 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T15:37:15,936 DEBUG [Thread-586 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T15:37:15,936 DEBUG [Thread-592 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T15:37:15,936 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48360, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T15:37:15,937 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7a780d55532c,32955,1731857800648, seqNum=-1] 2024-11-17T15:37:15,937 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T15:37:15,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] procedure2.ProcedureExecutor(1139): Stored pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees 2024-11-17T15:37:15,937 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T15:37:15,937 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48374, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T15:37:15,938 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7a780d55532c,32955,1731857800648, seqNum=-1] 2024-11-17T15:37:15,938 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7a780d55532c,32955,1731857800648, seqNum=-1] 2024-11-17T15:37:15,938 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T15:37:15,938 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T15:37:15,939 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T15:37:15,940 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T15:37:15,940 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48388, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T15:37:15,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-17T15:37:15,941 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5] 2024-11-17T15:37:15,941 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48392, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T15:37:15,941 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48396, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T15:37:15,942 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48408, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T15:37:15,942 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48420, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T15:37:15,942 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48428, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T15:37:15,942 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48438, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T15:37:15,942 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T15:37:15,942 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T15:37:15,943 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5] 2024-11-17T15:37:15,943 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5] 2024-11-17T15:37:15,943 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5] 2024-11-17T15:37:15,944 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5] 2024-11-17T15:37:15,944 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5] 2024-11-17T15:37:15,945 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5] 2024-11-17T15:37:15,945 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5] 2024-11-17T15:37:15,946 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5] 2024-11-17T15:37:15,967 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T15:37:15,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(8855): Flush requested on 34c72dd45fe3d9e6a1558a16ef53eb06 2024-11-17T15:37:15,968 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 34c72dd45fe3d9e6a1558a16ef53eb06 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-17T15:37:16,014 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:16,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48396 deadline: 1731857896007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:16,015 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:16,015 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:16,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48420 deadline: 1731857896008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:16,015 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:16,016 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:16,016 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:16,016 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:16,016 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:16,017 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:16,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48408 deadline: 1731857896011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:16,018 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:16,018 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:16,018 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:16,018 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:16,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48428 deadline: 1731857896012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:16,019 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:16,019 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:16,019 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:16,022 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:16,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48438 deadline: 1731857896014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:16,023 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:16,023 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:16,024 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:16,037 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111779f271a2368f42f3a0d5992831149892_34c72dd45fe3d9e6a1558a16ef53eb06 is 50, key is test_row_0/A:col10/1731857835966/Put/seqid=0 2024-11-17T15:37:16,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-17T15:37:16,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073741939_1115 (size=17034) 2024-11-17T15:37:16,047 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:16,054 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111779f271a2368f42f3a0d5992831149892_34c72dd45fe3d9e6a1558a16ef53eb06 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111779f271a2368f42f3a0d5992831149892_34c72dd45fe3d9e6a1558a16ef53eb06 2024-11-17T15:37:16,055 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/A/0d694a078a7e47c5bf8e05e16a9ed856, store: [table=TestAcidGuarantees family=A region=34c72dd45fe3d9e6a1558a16ef53eb06] 2024-11-17T15:37:16,064 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/A/0d694a078a7e47c5bf8e05e16a9ed856 is 175, key is test_row_0/A:col10/1731857835966/Put/seqid=0 2024-11-17T15:37:16,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073741940_1116 (size=48139) 2024-11-17T15:37:16,081 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/A/0d694a078a7e47c5bf8e05e16a9ed856 2024-11-17T15:37:16,099 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-17T15:37:16,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. 2024-11-17T15:37:16,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. as already flushing 2024-11-17T15:37:16,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. 2024-11-17T15:37:16,100 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 34c72dd45fe3d9e6a1558a16ef53eb06, NAME => 'TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:37:16,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 34c72dd45fe3d9e6a1558a16ef53eb06, NAME => 'TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:37:16,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 34c72dd45fe3d9e6a1558a16ef53eb06, NAME => 'TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 34c72dd45fe3d9e6a1558a16ef53eb06, NAME => 'TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:37:16,116 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/B/a08c5b3fe16349a2a54e268388f2b474 is 50, key is test_row_0/B:col10/1731857835966/Put/seqid=0 2024-11-17T15:37:16,127 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073741941_1117 (size=12001) 2024-11-17T15:37:16,128 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:16,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48396 deadline: 1731857896123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:16,129 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/B/a08c5b3fe16349a2a54e268388f2b474 2024-11-17T15:37:16,129 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:16,129 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:16,129 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:16,129 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:16,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48408 deadline: 1731857896124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:16,130 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:16,130 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:16,130 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:16,130 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:16,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48420 deadline: 1731857896124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:16,131 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:16,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48428 deadline: 1731857896124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:16,131 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:16,131 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:16,131 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:16,138 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:16,138 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:16,139 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:16,146 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:16,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48438 deadline: 1731857896142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:16,147 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:16,147 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:16,147 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:16,166 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/C/812b2c151b4a43569136b3f63286e434 is 50, key is test_row_0/C:col10/1731857835966/Put/seqid=0 2024-11-17T15:37:16,189 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073741942_1118 (size=12001) 2024-11-17T15:37:16,191 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/C/812b2c151b4a43569136b3f63286e434 2024-11-17T15:37:16,199 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/A/0d694a078a7e47c5bf8e05e16a9ed856 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/0d694a078a7e47c5bf8e05e16a9ed856 2024-11-17T15:37:16,208 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/0d694a078a7e47c5bf8e05e16a9ed856, entries=250, sequenceid=15, filesize=47.0 K 2024-11-17T15:37:16,209 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/B/a08c5b3fe16349a2a54e268388f2b474 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/B/a08c5b3fe16349a2a54e268388f2b474 2024-11-17T15:37:16,217 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/B/a08c5b3fe16349a2a54e268388f2b474, entries=150, sequenceid=15, filesize=11.7 K 2024-11-17T15:37:16,219 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/C/812b2c151b4a43569136b3f63286e434 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/C/812b2c151b4a43569136b3f63286e434 2024-11-17T15:37:16,225 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/C/812b2c151b4a43569136b3f63286e434, entries=150, sequenceid=15, filesize=11.7 K 2024-11-17T15:37:16,226 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 34c72dd45fe3d9e6a1558a16ef53eb06 in 259ms, sequenceid=15, compaction requested=false 2024-11-17T15:37:16,226 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-17T15:37:16,227 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 34c72dd45fe3d9e6a1558a16ef53eb06: 2024-11-17T15:37:16,253 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-17T15:37:16,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. 2024-11-17T15:37:16,253 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2902): Flushing 34c72dd45fe3d9e6a1558a16ef53eb06 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-17T15:37:16,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-17T15:37:16,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111709e98b3c6568449f8144b896d2e5bc20_34c72dd45fe3d9e6a1558a16ef53eb06 is 50, key is test_row_0/A:col10/1731857836008/Put/seqid=0 2024-11-17T15:37:16,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073741943_1119 (size=12154) 2024-11-17T15:37:16,345 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. as already flushing 2024-11-17T15:37:16,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(8855): Flush requested on 34c72dd45fe3d9e6a1558a16ef53eb06 2024-11-17T15:37:16,411 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:16,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48420 deadline: 1731857896396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:16,411 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:16,412 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:16,412 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:16,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48396 deadline: 1731857896397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:16,412 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:16,412 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:16,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48438 deadline: 1731857896402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:16,412 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:16,413 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:16,413 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:16,413 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:16,413 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:16,413 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:16,418 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:16,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48428 deadline: 1731857896412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:16,419 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:16,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48408 deadline: 1731857896413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:16,420 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:16,420 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:16,420 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:16,425 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:16,425 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:16,425 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:16,529 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:16,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48408 deadline: 1731857896524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:16,530 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:16,530 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:16,530 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:16,530 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:16,530 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:16,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48420 deadline: 1731857896524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:16,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48438 deadline: 1731857896525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:16,531 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:16,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48396 deadline: 1731857896525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:16,531 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:16,531 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:16,531 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:16,531 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:16,531 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:16,531 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:16,531 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:16,531 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:16,532 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:16,535 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:16,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48428 deadline: 1731857896533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:16,536 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:16,536 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:16,536 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:16,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-17T15:37:16,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:16,696 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111709e98b3c6568449f8144b896d2e5bc20_34c72dd45fe3d9e6a1558a16ef53eb06 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111709e98b3c6568449f8144b896d2e5bc20_34c72dd45fe3d9e6a1558a16ef53eb06 2024-11-17T15:37:16,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/A/62717ca05a3b4d2d8cce6517b08884a4, store: [table=TestAcidGuarantees family=A region=34c72dd45fe3d9e6a1558a16ef53eb06] 2024-11-17T15:37:16,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/A/62717ca05a3b4d2d8cce6517b08884a4 is 175, key is test_row_0/A:col10/1731857836008/Put/seqid=0 2024-11-17T15:37:16,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073741944_1120 (size=30955) 2024-11-17T15:37:16,716 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=40, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/A/62717ca05a3b4d2d8cce6517b08884a4 2024-11-17T15:37:16,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/B/78579c536eec4a0199fc0eb798ee7430 is 50, key is test_row_0/B:col10/1731857836008/Put/seqid=0 2024-11-17T15:37:16,734 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:16,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48408 deadline: 1731857896733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:16,735 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:16,735 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:16,735 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:16,746 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:16,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48420 deadline: 1731857896744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:16,747 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073741945_1121 (size=12001) 2024-11-17T15:37:16,747 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:16,747 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:16,747 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:16,747 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:16,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48428 deadline: 1731857896744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:16,748 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/B/78579c536eec4a0199fc0eb798ee7430 2024-11-17T15:37:16,748 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:16,748 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:16,748 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:16,748 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:16,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48438 deadline: 1731857896744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:16,749 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:16,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48396 deadline: 1731857896744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:16,749 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:16,749 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:16,749 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:16,751 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:16,751 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:16,751 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:16,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/C/aa994d1e00904e3380146931b98a05e6 is 50, key is test_row_0/C:col10/1731857836008/Put/seqid=0 2024-11-17T15:37:16,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073741946_1122 (size=12001) 2024-11-17T15:37:16,777 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/C/aa994d1e00904e3380146931b98a05e6 2024-11-17T15:37:16,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/A/62717ca05a3b4d2d8cce6517b08884a4 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/62717ca05a3b4d2d8cce6517b08884a4 2024-11-17T15:37:16,792 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/62717ca05a3b4d2d8cce6517b08884a4, entries=150, sequenceid=40, filesize=30.2 K 2024-11-17T15:37:16,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/B/78579c536eec4a0199fc0eb798ee7430 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/B/78579c536eec4a0199fc0eb798ee7430 2024-11-17T15:37:16,800 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/B/78579c536eec4a0199fc0eb798ee7430, entries=150, sequenceid=40, filesize=11.7 K 2024-11-17T15:37:16,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/C/aa994d1e00904e3380146931b98a05e6 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/C/aa994d1e00904e3380146931b98a05e6 2024-11-17T15:37:16,817 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/C/aa994d1e00904e3380146931b98a05e6, entries=150, sequenceid=40, filesize=11.7 K 2024-11-17T15:37:16,818 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 34c72dd45fe3d9e6a1558a16ef53eb06 in 565ms, sequenceid=40, compaction requested=false 2024-11-17T15:37:16,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2603): Flush status journal for 34c72dd45fe3d9e6a1558a16ef53eb06: 2024-11-17T15:37:16,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. 2024-11-17T15:37:16,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=41 2024-11-17T15:37:16,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.HMaster(4169): Remote procedure done, pid=41 2024-11-17T15:37:16,823 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=40 2024-11-17T15:37:16,823 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 878 msec 2024-11-17T15:37:16,826 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees in 890 msec 2024-11-17T15:37:17,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(8855): Flush requested on 34c72dd45fe3d9e6a1558a16ef53eb06 2024-11-17T15:37:17,045 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T15:37:17,045 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 34c72dd45fe3d9e6a1558a16ef53eb06 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-17T15:37:17,056 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111742437ab90ecd4ab0b4e46fe4b0612c1b_34c72dd45fe3d9e6a1558a16ef53eb06 is 50, key is test_row_0/A:col10/1731857836396/Put/seqid=0 2024-11-17T15:37:17,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-17T15:37:17,073 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T15:37:17,074 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T15:37:17,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-11-17T15:37:17,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-17T15:37:17,076 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T15:37:17,077 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T15:37:17,078 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T15:37:17,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073741947_1123 (size=14594) 2024-11-17T15:37:17,085 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:17,092 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111742437ab90ecd4ab0b4e46fe4b0612c1b_34c72dd45fe3d9e6a1558a16ef53eb06 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111742437ab90ecd4ab0b4e46fe4b0612c1b_34c72dd45fe3d9e6a1558a16ef53eb06 2024-11-17T15:37:17,094 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/A/1e36193653634d50a85aef94075acd11, store: [table=TestAcidGuarantees family=A region=34c72dd45fe3d9e6a1558a16ef53eb06] 2024-11-17T15:37:17,096 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/A/1e36193653634d50a85aef94075acd11 is 175, key is test_row_0/A:col10/1731857836396/Put/seqid=0 2024-11-17T15:37:17,098 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:17,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48420 deadline: 1731857897088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:17,099 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:17,099 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:17,099 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:17,099 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:17,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48428 deadline: 1731857897088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:17,099 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:17,099 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:17,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48438 deadline: 1731857897091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:17,099 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:17,099 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:17,099 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:17,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48396 deadline: 1731857897095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:17,100 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:17,100 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:17,100 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:17,100 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:17,100 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:17,100 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:17,100 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:17,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48408 deadline: 1731857897098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:17,101 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:17,101 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:17,101 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:17,117 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073741948_1124 (size=39549) 2024-11-17T15:37:17,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-17T15:37:17,204 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:17,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48428 deadline: 1731857897204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:17,205 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:17,205 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:17,205 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:17,205 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:17,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48438 deadline: 1731857897204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:17,206 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:17,206 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:17,206 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:17,206 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:17,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48408 deadline: 1731857897204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:17,207 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:17,207 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:17,207 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:17,207 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:17,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48420 deadline: 1731857897205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:17,207 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:17,207 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:17,207 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:17,207 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:17,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48396 deadline: 1731857897205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:17,208 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:17,208 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:17,208 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:17,231 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-17T15:37:17,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. 2024-11-17T15:37:17,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. as already flushing 2024-11-17T15:37:17,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. 2024-11-17T15:37:17,232 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 34c72dd45fe3d9e6a1558a16ef53eb06, NAME => 'TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:37:17,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 34c72dd45fe3d9e6a1558a16ef53eb06, NAME => 'TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:37:17,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 34c72dd45fe3d9e6a1558a16ef53eb06, NAME => 'TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 34c72dd45fe3d9e6a1558a16ef53eb06, NAME => 'TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:37:17,385 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-17T15:37:17,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. 2024-11-17T15:37:17,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. as already flushing 2024-11-17T15:37:17,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. 2024-11-17T15:37:17,385 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 34c72dd45fe3d9e6a1558a16ef53eb06, NAME => 'TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:37:17,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 34c72dd45fe3d9e6a1558a16ef53eb06, NAME => 'TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:37:17,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 34c72dd45fe3d9e6a1558a16ef53eb06, NAME => 'TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 34c72dd45fe3d9e6a1558a16ef53eb06, NAME => 'TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:37:17,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-17T15:37:17,416 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:17,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48428 deadline: 1731857897413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:17,416 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:17,416 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:17,416 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:17,416 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:17,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48438 deadline: 1731857897413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:17,417 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:17,417 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:17,417 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:17,417 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:17,417 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:17,417 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:17,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48408 deadline: 1731857897413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:17,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48420 deadline: 1731857897413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:17,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48396 deadline: 1731857897413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:17,418 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:17,418 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:17,418 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:17,418 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:17,418 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:17,418 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:17,418 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:17,418 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:17,418 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:17,518 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=52, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/A/1e36193653634d50a85aef94075acd11 2024-11-17T15:37:17,533 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/B/c5bb05e8ecc14638bfd65faddb999177 is 50, key is test_row_0/B:col10/1731857836396/Put/seqid=0 2024-11-17T15:37:17,537 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-17T15:37:17,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. 2024-11-17T15:37:17,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. as already flushing 2024-11-17T15:37:17,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. 2024-11-17T15:37:17,538 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 34c72dd45fe3d9e6a1558a16ef53eb06, NAME => 'TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:37:17,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 34c72dd45fe3d9e6a1558a16ef53eb06, NAME => 'TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:37:17,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 34c72dd45fe3d9e6a1558a16ef53eb06, NAME => 'TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 34c72dd45fe3d9e6a1558a16ef53eb06, NAME => 'TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:37:17,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073741949_1125 (size=12001) 2024-11-17T15:37:17,560 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/B/c5bb05e8ecc14638bfd65faddb999177 2024-11-17T15:37:17,582 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/C/3682a54890f34de09beaa648516b5f36 is 50, key is test_row_0/C:col10/1731857836396/Put/seqid=0 2024-11-17T15:37:17,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073741950_1126 (size=12001) 2024-11-17T15:37:17,600 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/C/3682a54890f34de09beaa648516b5f36 2024-11-17T15:37:17,605 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/A/1e36193653634d50a85aef94075acd11 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/1e36193653634d50a85aef94075acd11 2024-11-17T15:37:17,610 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-17T15:37:17,635 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/1e36193653634d50a85aef94075acd11, entries=200, sequenceid=52, filesize=38.6 K 2024-11-17T15:37:17,637 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/B/c5bb05e8ecc14638bfd65faddb999177 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/B/c5bb05e8ecc14638bfd65faddb999177 2024-11-17T15:37:17,644 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/B/c5bb05e8ecc14638bfd65faddb999177, entries=150, sequenceid=52, filesize=11.7 K 2024-11-17T15:37:17,646 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/C/3682a54890f34de09beaa648516b5f36 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/C/3682a54890f34de09beaa648516b5f36 2024-11-17T15:37:17,657 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/C/3682a54890f34de09beaa648516b5f36, entries=150, sequenceid=52, filesize=11.7 K 2024-11-17T15:37:17,659 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 34c72dd45fe3d9e6a1558a16ef53eb06 in 614ms, sequenceid=52, compaction requested=true 2024-11-17T15:37:17,659 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 34c72dd45fe3d9e6a1558a16ef53eb06: 2024-11-17T15:37:17,659 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 34c72dd45fe3d9e6a1558a16ef53eb06:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T15:37:17,659 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T15:37:17,659 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T15:37:17,659 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T15:37:17,659 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 34c72dd45fe3d9e6a1558a16ef53eb06:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T15:37:17,660 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T15:37:17,660 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 34c72dd45fe3d9e6a1558a16ef53eb06:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T15:37:17,660 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T15:37:17,660 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 118643 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T15:37:17,660 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T15:37:17,661 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1541): 34c72dd45fe3d9e6a1558a16ef53eb06/B is initiating minor compaction (all files) 2024-11-17T15:37:17,661 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1541): 34c72dd45fe3d9e6a1558a16ef53eb06/A is initiating minor compaction (all files) 2024-11-17T15:37:17,661 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 34c72dd45fe3d9e6a1558a16ef53eb06/A in TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. 2024-11-17T15:37:17,661 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 34c72dd45fe3d9e6a1558a16ef53eb06/B in TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. 2024-11-17T15:37:17,661 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/0d694a078a7e47c5bf8e05e16a9ed856, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/62717ca05a3b4d2d8cce6517b08884a4, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/1e36193653634d50a85aef94075acd11] into tmpdir=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp, totalSize=115.9 K 2024-11-17T15:37:17,661 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/B/a08c5b3fe16349a2a54e268388f2b474, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/B/78579c536eec4a0199fc0eb798ee7430, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/B/c5bb05e8ecc14638bfd65faddb999177] into tmpdir=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp, totalSize=35.2 K 2024-11-17T15:37:17,661 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. 2024-11-17T15:37:17,661 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting a08c5b3fe16349a2a54e268388f2b474, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731857835946 2024-11-17T15:37:17,661 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. files: [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/0d694a078a7e47c5bf8e05e16a9ed856, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/62717ca05a3b4d2d8cce6517b08884a4, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/1e36193653634d50a85aef94075acd11] 2024-11-17T15:37:17,662 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0d694a078a7e47c5bf8e05e16a9ed856, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731857835946 2024-11-17T15:37:17,662 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting 78579c536eec4a0199fc0eb798ee7430, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731857836006 2024-11-17T15:37:17,669 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 62717ca05a3b4d2d8cce6517b08884a4, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731857836006 2024-11-17T15:37:17,669 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting c5bb05e8ecc14638bfd65faddb999177, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731857836394 2024-11-17T15:37:17,670 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1e36193653634d50a85aef94075acd11, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731857836394 2024-11-17T15:37:17,682 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 34c72dd45fe3d9e6a1558a16ef53eb06#B#compaction#108 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T15:37:17,682 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/B/fec8744dafb14d29af8aaa45211efa1f is 50, key is test_row_0/B:col10/1731857836396/Put/seqid=0 2024-11-17T15:37:17,689 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=34c72dd45fe3d9e6a1558a16ef53eb06] 2024-11-17T15:37:17,692 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-17T15:37:17,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. 2024-11-17T15:37:17,692 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing 34c72dd45fe3d9e6a1558a16ef53eb06 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-17T15:37:17,703 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241117cbe2e4f134fc4c30ab7213de1985c0a4_34c72dd45fe3d9e6a1558a16ef53eb06 store=[table=TestAcidGuarantees family=A region=34c72dd45fe3d9e6a1558a16ef53eb06] 2024-11-17T15:37:17,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-17T15:37:17,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073741951_1127 (size=12104) 2024-11-17T15:37:17,710 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241117cbe2e4f134fc4c30ab7213de1985c0a4_34c72dd45fe3d9e6a1558a16ef53eb06, store=[table=TestAcidGuarantees family=A region=34c72dd45fe3d9e6a1558a16ef53eb06] 2024-11-17T15:37:17,710 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117cbe2e4f134fc4c30ab7213de1985c0a4_34c72dd45fe3d9e6a1558a16ef53eb06 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=34c72dd45fe3d9e6a1558a16ef53eb06] 2024-11-17T15:37:17,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411175ea7656ae19c4c82bc00b23a4044f4cd_34c72dd45fe3d9e6a1558a16ef53eb06 is 50, key is test_row_0/A:col10/1731857837087/Put/seqid=0 2024-11-17T15:37:17,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(8855): Flush requested on 34c72dd45fe3d9e6a1558a16ef53eb06 2024-11-17T15:37:17,736 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. as already flushing 2024-11-17T15:37:17,744 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:17,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48396 deadline: 1731857897735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:17,746 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:17,746 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:17,746 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:17,746 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:17,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48438 deadline: 1731857897737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:17,747 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:17,744 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:17,747 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:17,747 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:17,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48428 deadline: 1731857897735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:17,747 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:17,747 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:17,748 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:17,748 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073741952_1128 (size=4469) 2024-11-17T15:37:17,754 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:17,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48408 deadline: 1731857897744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:17,755 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:17,755 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:17,755 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:17,755 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:17,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48420 deadline: 1731857897744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:17,755 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:17,756 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:17,756 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:17,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073741953_1129 (size=12154) 2024-11-17T15:37:17,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:17,772 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411175ea7656ae19c4c82bc00b23a4044f4cd_34c72dd45fe3d9e6a1558a16ef53eb06 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411175ea7656ae19c4c82bc00b23a4044f4cd_34c72dd45fe3d9e6a1558a16ef53eb06 2024-11-17T15:37:17,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/A/c3d4932a92e34b278e3dedfbef004167, store: [table=TestAcidGuarantees family=A region=34c72dd45fe3d9e6a1558a16ef53eb06] 2024-11-17T15:37:17,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/A/c3d4932a92e34b278e3dedfbef004167 is 175, key is test_row_0/A:col10/1731857837087/Put/seqid=0 2024-11-17T15:37:17,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073741954_1130 (size=30955) 2024-11-17T15:37:17,796 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=76, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/A/c3d4932a92e34b278e3dedfbef004167 2024-11-17T15:37:17,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/B/fb14878cf5c2491f849182859f0b4a53 is 50, key is test_row_0/B:col10/1731857837087/Put/seqid=0 2024-11-17T15:37:17,839 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073741955_1131 (size=12001) 2024-11-17T15:37:17,842 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/B/fb14878cf5c2491f849182859f0b4a53 2024-11-17T15:37:17,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/C/7b2d694a9e79450585cbd56602818b68 is 50, key is test_row_0/C:col10/1731857837087/Put/seqid=0 2024-11-17T15:37:17,855 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:17,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48438 deadline: 1731857897854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:17,856 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:17,856 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:17,856 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:17,857 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:17,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48396 deadline: 1731857897854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:17,857 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:17,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48428 deadline: 1731857897854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:17,858 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:17,858 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:17,858 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:17,858 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:17,858 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:17,858 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:17,865 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:17,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48408 deadline: 1731857897863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:17,865 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:17,865 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:17,865 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:17,866 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:17,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48420 deadline: 1731857897863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:17,866 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:17,866 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:17,866 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:17,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073741956_1132 (size=12001) 2024-11-17T15:37:17,879 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/C/7b2d694a9e79450585cbd56602818b68 2024-11-17T15:37:17,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/A/c3d4932a92e34b278e3dedfbef004167 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/c3d4932a92e34b278e3dedfbef004167 2024-11-17T15:37:17,895 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/c3d4932a92e34b278e3dedfbef004167, entries=150, sequenceid=76, filesize=30.2 K 2024-11-17T15:37:17,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/B/fb14878cf5c2491f849182859f0b4a53 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/B/fb14878cf5c2491f849182859f0b4a53 2024-11-17T15:37:17,903 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/B/fb14878cf5c2491f849182859f0b4a53, entries=150, sequenceid=76, filesize=11.7 K 2024-11-17T15:37:17,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/C/7b2d694a9e79450585cbd56602818b68 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/C/7b2d694a9e79450585cbd56602818b68 2024-11-17T15:37:17,912 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/C/7b2d694a9e79450585cbd56602818b68, entries=150, sequenceid=76, filesize=11.7 K 2024-11-17T15:37:17,914 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 34c72dd45fe3d9e6a1558a16ef53eb06 in 222ms, sequenceid=76, compaction requested=true 2024-11-17T15:37:17,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for 34c72dd45fe3d9e6a1558a16ef53eb06: 2024-11-17T15:37:17,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. 2024-11-17T15:37:17,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-11-17T15:37:17,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-11-17T15:37:17,919 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-11-17T15:37:17,919 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 840 msec 2024-11-17T15:37:17,922 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 846 msec 2024-11-17T15:37:18,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(8855): Flush requested on 34c72dd45fe3d9e6a1558a16ef53eb06 2024-11-17T15:37:18,066 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T15:37:18,066 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 34c72dd45fe3d9e6a1558a16ef53eb06 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-17T15:37:18,079 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411174e968bdcd9674cee844a34e18efab779_34c72dd45fe3d9e6a1558a16ef53eb06 is 50, key is test_row_0/A:col10/1731857838064/Put/seqid=0 2024-11-17T15:37:18,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073741957_1133 (size=14594) 2024-11-17T15:37:18,094 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:18,103 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411174e968bdcd9674cee844a34e18efab779_34c72dd45fe3d9e6a1558a16ef53eb06 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411174e968bdcd9674cee844a34e18efab779_34c72dd45fe3d9e6a1558a16ef53eb06 2024-11-17T15:37:18,105 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/A/f6b0cf725d71426aa52d8bfea323007b, store: [table=TestAcidGuarantees family=A region=34c72dd45fe3d9e6a1558a16ef53eb06] 2024-11-17T15:37:18,107 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/A/f6b0cf725d71426aa52d8bfea323007b is 175, key is test_row_0/A:col10/1731857838064/Put/seqid=0 2024-11-17T15:37:18,114 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/B/fec8744dafb14d29af8aaa45211efa1f as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/B/fec8744dafb14d29af8aaa45211efa1f 2024-11-17T15:37:18,118 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:18,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48408 deadline: 1731857898107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:18,119 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:18,119 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:18,119 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:18,124 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 34c72dd45fe3d9e6a1558a16ef53eb06/B of 34c72dd45fe3d9e6a1558a16ef53eb06 into fec8744dafb14d29af8aaa45211efa1f(size=11.8 K), total size for store is 23.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T15:37:18,124 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 34c72dd45fe3d9e6a1558a16ef53eb06: 2024-11-17T15:37:18,124 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., storeName=34c72dd45fe3d9e6a1558a16ef53eb06/B, priority=13, startTime=1731857837659; duration=0sec 2024-11-17T15:37:18,124 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T15:37:18,124 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 34c72dd45fe3d9e6a1558a16ef53eb06:B 2024-11-17T15:37:18,124 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T15:37:18,126 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:18,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48396 deadline: 1731857898110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:18,126 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:18,126 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:18,127 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:18,127 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:18,127 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T15:37:18,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48420 deadline: 1731857898110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:18,127 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1541): 34c72dd45fe3d9e6a1558a16ef53eb06/C is initiating minor compaction (all files) 2024-11-17T15:37:18,127 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 34c72dd45fe3d9e6a1558a16ef53eb06/C in TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. 2024-11-17T15:37:18,127 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/C/812b2c151b4a43569136b3f63286e434, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/C/aa994d1e00904e3380146931b98a05e6, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/C/3682a54890f34de09beaa648516b5f36, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/C/7b2d694a9e79450585cbd56602818b68] into tmpdir=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp, totalSize=46.9 K 2024-11-17T15:37:18,127 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:18,127 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:18,127 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:18,127 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:18,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48428 deadline: 1731857898116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:18,128 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:18,128 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:18,128 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:18,128 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:18,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48438 deadline: 1731857898118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:18,129 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:18,129 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:18,129 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:18,129 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting 812b2c151b4a43569136b3f63286e434, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731857835946 2024-11-17T15:37:18,130 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting aa994d1e00904e3380146931b98a05e6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731857836006 2024-11-17T15:37:18,130 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting 3682a54890f34de09beaa648516b5f36, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731857836394 2024-11-17T15:37:18,131 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting 7b2d694a9e79450585cbd56602818b68, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731857837085 2024-11-17T15:37:18,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073741958_1134 (size=39549) 2024-11-17T15:37:18,146 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=89, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/A/f6b0cf725d71426aa52d8bfea323007b 2024-11-17T15:37:18,146 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 34c72dd45fe3d9e6a1558a16ef53eb06#C#compaction#114 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T15:37:18,147 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/C/f0d3442a16ba4f339f2f4bfb8141ba32 is 50, key is test_row_0/C:col10/1731857837087/Put/seqid=0 2024-11-17T15:37:18,150 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 34c72dd45fe3d9e6a1558a16ef53eb06#A#compaction#109 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T15:37:18,151 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/A/1cb34538f39a4907a01461f0c2e76efc is 175, key is test_row_0/A:col10/1731857836396/Put/seqid=0 2024-11-17T15:37:18,171 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/B/542cf35da1554b538c155e8530d86d40 is 50, key is test_row_0/B:col10/1731857838064/Put/seqid=0 2024-11-17T15:37:18,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073741959_1135 (size=31058) 2024-11-17T15:37:18,191 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/A/1cb34538f39a4907a01461f0c2e76efc as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/1cb34538f39a4907a01461f0c2e76efc 2024-11-17T15:37:18,201 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 34c72dd45fe3d9e6a1558a16ef53eb06/A of 34c72dd45fe3d9e6a1558a16ef53eb06 into 1cb34538f39a4907a01461f0c2e76efc(size=30.3 K), total size for store is 60.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T15:37:18,201 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 34c72dd45fe3d9e6a1558a16ef53eb06: 2024-11-17T15:37:18,201 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., storeName=34c72dd45fe3d9e6a1558a16ef53eb06/A, priority=13, startTime=1731857837659; duration=0sec 2024-11-17T15:37:18,202 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T15:37:18,202 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 34c72dd45fe3d9e6a1558a16ef53eb06:A 2024-11-17T15:37:18,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073741961_1137 (size=12001) 2024-11-17T15:37:18,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073741960_1136 (size=12139) 2024-11-17T15:37:18,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-17T15:37:18,213 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T15:37:18,217 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/C/f0d3442a16ba4f339f2f4bfb8141ba32 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/C/f0d3442a16ba4f339f2f4bfb8141ba32 2024-11-17T15:37:18,218 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T15:37:18,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-11-17T15:37:18,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-17T15:37:18,223 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T15:37:18,223 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:18,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48408 deadline: 1731857898223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:18,224 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T15:37:18,225 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:18,225 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T15:37:18,225 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:18,225 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:18,225 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 34c72dd45fe3d9e6a1558a16ef53eb06/C of 34c72dd45fe3d9e6a1558a16ef53eb06 into f0d3442a16ba4f339f2f4bfb8141ba32(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T15:37:18,225 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 34c72dd45fe3d9e6a1558a16ef53eb06: 2024-11-17T15:37:18,225 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., storeName=34c72dd45fe3d9e6a1558a16ef53eb06/C, priority=12, startTime=1731857837660; duration=0sec 2024-11-17T15:37:18,226 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T15:37:18,226 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 34c72dd45fe3d9e6a1558a16ef53eb06:C 2024-11-17T15:37:18,240 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:18,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48396 deadline: 1731857898234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:18,241 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:18,241 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:18,241 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:18,241 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:18,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48428 deadline: 1731857898234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:18,241 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:18,241 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:18,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48438 deadline: 1731857898234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:18,241 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:18,242 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:18,241 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:18,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48420 deadline: 1731857898235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:18,242 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:18,242 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:18,242 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:18,243 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:18,243 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:18,243 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:18,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-17T15:37:18,379 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-17T15:37:18,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. 2024-11-17T15:37:18,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. as already flushing 2024-11-17T15:37:18,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. 2024-11-17T15:37:18,380 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 34c72dd45fe3d9e6a1558a16ef53eb06, NAME => 'TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:37:18,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 34c72dd45fe3d9e6a1558a16ef53eb06, NAME => 'TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:37:18,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 34c72dd45fe3d9e6a1558a16ef53eb06, NAME => 'TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 34c72dd45fe3d9e6a1558a16ef53eb06, NAME => 'TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:37:18,436 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:18,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48408 deadline: 1731857898433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:18,437 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:18,437 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:18,437 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:18,455 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:18,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48396 deadline: 1731857898453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:18,456 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:18,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48428 deadline: 1731857898453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:18,456 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:18,456 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:18,456 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:18,456 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:18,456 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:18,456 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:18,456 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:18,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48420 deadline: 1731857898453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:18,457 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:18,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48438 deadline: 1731857898454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:18,457 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:18,457 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:18,457 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:18,457 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:18,457 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:18,457 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:18,534 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-17T15:37:18,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. 2024-11-17T15:37:18,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. as already flushing 2024-11-17T15:37:18,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. 2024-11-17T15:37:18,534 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 34c72dd45fe3d9e6a1558a16ef53eb06, NAME => 'TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:37:18,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 34c72dd45fe3d9e6a1558a16ef53eb06, NAME => 'TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:37:18,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 34c72dd45fe3d9e6a1558a16ef53eb06, NAME => 'TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 34c72dd45fe3d9e6a1558a16ef53eb06, NAME => 'TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:37:18,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-17T15:37:18,610 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=89 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/B/542cf35da1554b538c155e8530d86d40 2024-11-17T15:37:18,641 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/C/a8935481cbc74a389c98c34b0650c8d3 is 50, key is test_row_0/C:col10/1731857838064/Put/seqid=0 2024-11-17T15:37:18,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073741962_1138 (size=12001) 2024-11-17T15:37:18,688 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-17T15:37:18,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. 2024-11-17T15:37:18,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. as already flushing 2024-11-17T15:37:18,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. 2024-11-17T15:37:18,689 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 34c72dd45fe3d9e6a1558a16ef53eb06, NAME => 'TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:37:18,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 34c72dd45fe3d9e6a1558a16ef53eb06, NAME => 'TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:37:18,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 34c72dd45fe3d9e6a1558a16ef53eb06, NAME => 'TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 34c72dd45fe3d9e6a1558a16ef53eb06, NAME => 'TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:37:18,748 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:18,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48408 deadline: 1731857898743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:18,748 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:18,748 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:18,748 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:18,764 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:18,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48396 deadline: 1731857898764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:18,765 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:18,765 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:18,765 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:18,765 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:18,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48428 deadline: 1731857898764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:18,765 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:18,765 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:18,765 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:18,765 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:18,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48420 deadline: 1731857898764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:18,766 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:18,766 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:18,766 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:18,766 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:18,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48438 deadline: 1731857898764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:18,767 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:18,767 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:18,767 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:18,842 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-17T15:37:18,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. 2024-11-17T15:37:18,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. as already flushing 2024-11-17T15:37:18,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. 2024-11-17T15:37:18,843 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 34c72dd45fe3d9e6a1558a16ef53eb06, NAME => 'TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:37:18,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 34c72dd45fe3d9e6a1558a16ef53eb06, NAME => 'TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:37:18,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 34c72dd45fe3d9e6a1558a16ef53eb06, NAME => 'TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 34c72dd45fe3d9e6a1558a16ef53eb06, NAME => 'TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:37:18,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-17T15:37:19,000 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-17T15:37:19,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. 2024-11-17T15:37:19,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. as already flushing 2024-11-17T15:37:19,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. 2024-11-17T15:37:19,001 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 34c72dd45fe3d9e6a1558a16ef53eb06, NAME => 'TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:37:19,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 34c72dd45fe3d9e6a1558a16ef53eb06, NAME => 'TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:37:19,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 34c72dd45fe3d9e6a1558a16ef53eb06, NAME => 'TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 34c72dd45fe3d9e6a1558a16ef53eb06, NAME => 'TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:37:19,057 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=89 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/C/a8935481cbc74a389c98c34b0650c8d3 2024-11-17T15:37:19,063 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/A/f6b0cf725d71426aa52d8bfea323007b as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/f6b0cf725d71426aa52d8bfea323007b 2024-11-17T15:37:19,067 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/f6b0cf725d71426aa52d8bfea323007b, entries=200, sequenceid=89, filesize=38.6 K 2024-11-17T15:37:19,068 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/B/542cf35da1554b538c155e8530d86d40 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/B/542cf35da1554b538c155e8530d86d40 2024-11-17T15:37:19,073 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/B/542cf35da1554b538c155e8530d86d40, entries=150, sequenceid=89, filesize=11.7 K 2024-11-17T15:37:19,074 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/C/a8935481cbc74a389c98c34b0650c8d3 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/C/a8935481cbc74a389c98c34b0650c8d3 2024-11-17T15:37:19,080 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/C/a8935481cbc74a389c98c34b0650c8d3, entries=150, sequenceid=89, filesize=11.7 K 2024-11-17T15:37:19,081 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 34c72dd45fe3d9e6a1558a16ef53eb06 in 1015ms, sequenceid=89, compaction requested=true 2024-11-17T15:37:19,082 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 34c72dd45fe3d9e6a1558a16ef53eb06: 2024-11-17T15:37:19,082 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T15:37:19,083 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 34c72dd45fe3d9e6a1558a16ef53eb06:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T15:37:19,083 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T15:37:19,083 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 34c72dd45fe3d9e6a1558a16ef53eb06:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T15:37:19,083 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T15:37:19,083 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T15:37:19,083 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 34c72dd45fe3d9e6a1558a16ef53eb06:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T15:37:19,083 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T15:37:19,083 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101562 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T15:37:19,083 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1541): 34c72dd45fe3d9e6a1558a16ef53eb06/A is initiating minor compaction (all files) 2024-11-17T15:37:19,084 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 34c72dd45fe3d9e6a1558a16ef53eb06/A in TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. 2024-11-17T15:37:19,084 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/1cb34538f39a4907a01461f0c2e76efc, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/c3d4932a92e34b278e3dedfbef004167, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/f6b0cf725d71426aa52d8bfea323007b] into tmpdir=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp, totalSize=99.2 K 2024-11-17T15:37:19,084 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. 2024-11-17T15:37:19,084 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. files: [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/1cb34538f39a4907a01461f0c2e76efc, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/c3d4932a92e34b278e3dedfbef004167, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/f6b0cf725d71426aa52d8bfea323007b] 2024-11-17T15:37:19,084 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1cb34538f39a4907a01461f0c2e76efc, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731857836394 2024-11-17T15:37:19,084 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T15:37:19,084 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1541): 34c72dd45fe3d9e6a1558a16ef53eb06/B is initiating minor compaction (all files) 2024-11-17T15:37:19,085 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 34c72dd45fe3d9e6a1558a16ef53eb06/B in TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. 2024-11-17T15:37:19,085 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting c3d4932a92e34b278e3dedfbef004167, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731857837085 2024-11-17T15:37:19,085 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/B/fec8744dafb14d29af8aaa45211efa1f, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/B/fb14878cf5c2491f849182859f0b4a53, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/B/542cf35da1554b538c155e8530d86d40] into tmpdir=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp, totalSize=35.3 K 2024-11-17T15:37:19,085 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting f6b0cf725d71426aa52d8bfea323007b, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1731857837731 2024-11-17T15:37:19,085 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting fec8744dafb14d29af8aaa45211efa1f, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731857836394 2024-11-17T15:37:19,086 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting fb14878cf5c2491f849182859f0b4a53, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731857837085 2024-11-17T15:37:19,086 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting 542cf35da1554b538c155e8530d86d40, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1731857837731 2024-11-17T15:37:19,093 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=34c72dd45fe3d9e6a1558a16ef53eb06] 2024-11-17T15:37:19,096 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241117d78a8802a4d54328accce383ade169ed_34c72dd45fe3d9e6a1558a16ef53eb06 store=[table=TestAcidGuarantees family=A region=34c72dd45fe3d9e6a1558a16ef53eb06] 2024-11-17T15:37:19,097 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 34c72dd45fe3d9e6a1558a16ef53eb06#B#compaction#118 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T15:37:19,098 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/B/35379b7f83784fdf90a10bce4fd4850c is 50, key is test_row_0/B:col10/1731857838064/Put/seqid=0 2024-11-17T15:37:19,100 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241117d78a8802a4d54328accce383ade169ed_34c72dd45fe3d9e6a1558a16ef53eb06, store=[table=TestAcidGuarantees family=A region=34c72dd45fe3d9e6a1558a16ef53eb06] 2024-11-17T15:37:19,101 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117d78a8802a4d54328accce383ade169ed_34c72dd45fe3d9e6a1558a16ef53eb06 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=34c72dd45fe3d9e6a1558a16ef53eb06] 2024-11-17T15:37:19,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073741963_1139 (size=12207) 2024-11-17T15:37:19,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073741964_1140 (size=4469) 2024-11-17T15:37:19,145 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 34c72dd45fe3d9e6a1558a16ef53eb06#A#compaction#117 average throughput is 0.47 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T15:37:19,146 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/A/46801bd954344096976cf0e0ae332a6e is 175, key is test_row_0/A:col10/1731857838064/Put/seqid=0 2024-11-17T15:37:19,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073741965_1141 (size=31161) 2024-11-17T15:37:19,154 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-17T15:37:19,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. 2024-11-17T15:37:19,154 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing 34c72dd45fe3d9e6a1558a16ef53eb06 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-17T15:37:19,159 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/A/46801bd954344096976cf0e0ae332a6e as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/46801bd954344096976cf0e0ae332a6e 2024-11-17T15:37:19,167 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 34c72dd45fe3d9e6a1558a16ef53eb06/A of 34c72dd45fe3d9e6a1558a16ef53eb06 into 46801bd954344096976cf0e0ae332a6e(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T15:37:19,167 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 34c72dd45fe3d9e6a1558a16ef53eb06: 2024-11-17T15:37:19,167 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., storeName=34c72dd45fe3d9e6a1558a16ef53eb06/A, priority=13, startTime=1731857839082; duration=0sec 2024-11-17T15:37:19,167 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T15:37:19,167 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 34c72dd45fe3d9e6a1558a16ef53eb06:A 2024-11-17T15:37:19,167 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-17T15:37:19,168 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-17T15:37:19,168 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-17T15:37:19,168 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. because compaction request was cancelled 2024-11-17T15:37:19,168 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 34c72dd45fe3d9e6a1558a16ef53eb06:C 2024-11-17T15:37:19,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117f2138863267342a08d6776b42ff12576_34c72dd45fe3d9e6a1558a16ef53eb06 is 50, key is test_row_0/A:col10/1731857838113/Put/seqid=0 2024-11-17T15:37:19,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073741966_1142 (size=12154) 2024-11-17T15:37:19,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(8855): Flush requested on 34c72dd45fe3d9e6a1558a16ef53eb06 2024-11-17T15:37:19,255 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. as already flushing 2024-11-17T15:37:19,295 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:19,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48438 deadline: 1731857899285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:19,296 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:19,296 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:19,296 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:19,296 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:19,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48428 deadline: 1731857899285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:19,296 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:19,296 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:19,297 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:19,297 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:19,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48408 deadline: 1731857899286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:19,298 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:19,298 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:19,298 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:19,298 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:19,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48396 deadline: 1731857899289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:19,299 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:19,299 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:19,299 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:19,299 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:19,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48420 deadline: 1731857899293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:19,300 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:19,300 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:19,300 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:19,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-17T15:37:19,407 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:19,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48438 deadline: 1731857899403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:19,408 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:19,408 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:19,408 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:19,408 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:19,408 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:19,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48428 deadline: 1731857899403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:19,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48396 deadline: 1731857899403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:19,409 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:19,409 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:19,409 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:19,409 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:19,409 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:19,409 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:19,409 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:19,409 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:19,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48420 deadline: 1731857899404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:19,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48408 deadline: 1731857899403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:19,410 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:19,410 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:19,410 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:19,410 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:19,410 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:19,411 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:19,532 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/B/35379b7f83784fdf90a10bce4fd4850c as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/B/35379b7f83784fdf90a10bce4fd4850c 2024-11-17T15:37:19,539 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 34c72dd45fe3d9e6a1558a16ef53eb06/B of 34c72dd45fe3d9e6a1558a16ef53eb06 into 35379b7f83784fdf90a10bce4fd4850c(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T15:37:19,539 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 34c72dd45fe3d9e6a1558a16ef53eb06: 2024-11-17T15:37:19,539 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., storeName=34c72dd45fe3d9e6a1558a16ef53eb06/B, priority=13, startTime=1731857839083; duration=0sec 2024-11-17T15:37:19,539 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T15:37:19,539 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 34c72dd45fe3d9e6a1558a16ef53eb06:B 2024-11-17T15:37:19,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:19,600 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117f2138863267342a08d6776b42ff12576_34c72dd45fe3d9e6a1558a16ef53eb06 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117f2138863267342a08d6776b42ff12576_34c72dd45fe3d9e6a1558a16ef53eb06 2024-11-17T15:37:19,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/A/b6bfd52673314433b51ea810c706b8cf, store: [table=TestAcidGuarantees family=A region=34c72dd45fe3d9e6a1558a16ef53eb06] 2024-11-17T15:37:19,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/A/b6bfd52673314433b51ea810c706b8cf is 175, key is test_row_0/A:col10/1731857838113/Put/seqid=0 2024-11-17T15:37:19,619 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:19,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48438 deadline: 1731857899614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:19,620 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:19,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48420 deadline: 1731857899614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:19,620 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:19,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48428 deadline: 1731857899614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:19,620 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:19,620 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:19,621 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:19,621 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:19,621 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:19,621 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:19,621 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:19,621 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:19,621 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:19,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48396 deadline: 1731857899614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:19,621 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:19,621 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:19,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48408 deadline: 1731857899614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:19,622 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:19,622 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:19,622 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:19,622 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:19,622 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:19,622 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:19,632 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073741967_1143 (size=30955) 2024-11-17T15:37:19,633 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=115, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/A/b6bfd52673314433b51ea810c706b8cf 2024-11-17T15:37:19,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/B/ddf4281b810a4760a3dc29ab840b5225 is 50, key is test_row_0/B:col10/1731857838113/Put/seqid=0 2024-11-17T15:37:19,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073741968_1144 (size=12001) 2024-11-17T15:37:19,668 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/B/ddf4281b810a4760a3dc29ab840b5225 2024-11-17T15:37:19,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/C/a792eefe4ce54c548f3fb56f254f0321 is 50, key is test_row_0/C:col10/1731857838113/Put/seqid=0 2024-11-17T15:37:19,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073741969_1145 (size=12001) 2024-11-17T15:37:19,685 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/C/a792eefe4ce54c548f3fb56f254f0321 2024-11-17T15:37:19,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/A/b6bfd52673314433b51ea810c706b8cf as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/b6bfd52673314433b51ea810c706b8cf 2024-11-17T15:37:19,696 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/b6bfd52673314433b51ea810c706b8cf, entries=150, sequenceid=115, filesize=30.2 K 2024-11-17T15:37:19,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/B/ddf4281b810a4760a3dc29ab840b5225 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/B/ddf4281b810a4760a3dc29ab840b5225 2024-11-17T15:37:19,702 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/B/ddf4281b810a4760a3dc29ab840b5225, entries=150, sequenceid=115, filesize=11.7 K 2024-11-17T15:37:19,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/C/a792eefe4ce54c548f3fb56f254f0321 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/C/a792eefe4ce54c548f3fb56f254f0321 2024-11-17T15:37:19,708 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/C/a792eefe4ce54c548f3fb56f254f0321, entries=150, sequenceid=115, filesize=11.7 K 2024-11-17T15:37:19,709 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=73.80 KB/75570 for 34c72dd45fe3d9e6a1558a16ef53eb06 in 555ms, sequenceid=115, compaction requested=true 2024-11-17T15:37:19,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for 34c72dd45fe3d9e6a1558a16ef53eb06: 2024-11-17T15:37:19,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. 2024-11-17T15:37:19,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-11-17T15:37:19,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-11-17T15:37:19,715 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-11-17T15:37:19,715 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4860 sec 2024-11-17T15:37:19,718 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 1.4970 sec 2024-11-17T15:37:19,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(8855): Flush requested on 34c72dd45fe3d9e6a1558a16ef53eb06 2024-11-17T15:37:19,927 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T15:37:19,927 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 34c72dd45fe3d9e6a1558a16ef53eb06 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-17T15:37:19,955 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111782434585db8044cb9a8cef2199d06bc5_34c72dd45fe3d9e6a1558a16ef53eb06 is 50, key is test_row_0/A:col10/1731857839926/Put/seqid=0 2024-11-17T15:37:19,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073741970_1146 (size=12254) 2024-11-17T15:37:20,001 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:20,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48428 deadline: 1731857899965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:20,002 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:20,002 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:20,002 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:20,006 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:20,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48438 deadline: 1731857899997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:20,007 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:20,007 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:20,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48420 deadline: 1731857899997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:20,007 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:20,007 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:20,007 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:20,007 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:20,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48408 deadline: 1731857899998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:20,007 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:20,007 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:20,008 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:20,008 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:20,008 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:20,008 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:20,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48396 deadline: 1731857899999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:20,008 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:20,008 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:20,008 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:20,117 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:20,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48428 deadline: 1731857900113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:20,117 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:20,118 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:20,118 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:20,118 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:20,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48420 deadline: 1731857900113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:20,118 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:20,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48438 deadline: 1731857900113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:20,118 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:20,118 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:20,118 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:20,118 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:20,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48408 deadline: 1731857900113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:20,118 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:20,118 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:20,118 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:20,118 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:20,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48396 deadline: 1731857900113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:20,119 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:20,119 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:20,119 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:20,119 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:20,119 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:20,120 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:20,326 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:20,326 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:20,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48428 deadline: 1731857900323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:20,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48420 deadline: 1731857900323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:20,327 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:20,327 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:20,327 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:20,327 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:20,327 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:20,327 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:20,327 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:20,327 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:20,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48438 deadline: 1731857900323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:20,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48408 deadline: 1731857900323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:20,328 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:20,328 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:20,328 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:20,328 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:20,328 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:20,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48396 deadline: 1731857900323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:20,328 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:20,328 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:20,329 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:20,329 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:20,329 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:20,371 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:20,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-17T15:37:20,373 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T15:37:20,374 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T15:37:20,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-11-17T15:37:20,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-17T15:37:20,377 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T15:37:20,380 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T15:37:20,381 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T15:37:20,384 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111782434585db8044cb9a8cef2199d06bc5_34c72dd45fe3d9e6a1558a16ef53eb06 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111782434585db8044cb9a8cef2199d06bc5_34c72dd45fe3d9e6a1558a16ef53eb06 2024-11-17T15:37:20,385 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/A/42d8a488421545ed8912eb775fd9caaa, store: [table=TestAcidGuarantees family=A region=34c72dd45fe3d9e6a1558a16ef53eb06] 2024-11-17T15:37:20,386 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/A/42d8a488421545ed8912eb775fd9caaa is 175, key is test_row_0/A:col10/1731857839926/Put/seqid=0 2024-11-17T15:37:20,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073741971_1147 (size=31055) 2024-11-17T15:37:20,419 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=132, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/A/42d8a488421545ed8912eb775fd9caaa 2024-11-17T15:37:20,434 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/B/4d02db3f5f28436580fac6958e3bb965 is 50, key is test_row_0/B:col10/1731857839926/Put/seqid=0 2024-11-17T15:37:20,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073741972_1148 (size=12101) 2024-11-17T15:37:20,479 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/B/4d02db3f5f28436580fac6958e3bb965 2024-11-17T15:37:20,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-17T15:37:20,490 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/C/7f59a4ccad1543c4a2e67a41e77bce52 is 50, key is test_row_0/C:col10/1731857839926/Put/seqid=0 2024-11-17T15:37:20,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073741973_1149 (size=12101) 2024-11-17T15:37:20,535 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-17T15:37:20,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. 2024-11-17T15:37:20,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. as already flushing 2024-11-17T15:37:20,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. 2024-11-17T15:37:20,536 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 34c72dd45fe3d9e6a1558a16ef53eb06, NAME => 'TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:37:20,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 34c72dd45fe3d9e6a1558a16ef53eb06, NAME => 'TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:37:20,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 34c72dd45fe3d9e6a1558a16ef53eb06, NAME => 'TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 34c72dd45fe3d9e6a1558a16ef53eb06, NAME => 'TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:37:20,635 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:20,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48420 deadline: 1731857900634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:20,635 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:20,635 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:20,635 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:20,635 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:20,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48438 deadline: 1731857900634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:20,636 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:20,636 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:20,636 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:20,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48428 deadline: 1731857900634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:20,636 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:20,636 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:20,636 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:20,636 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:20,636 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:20,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48408 deadline: 1731857900634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:20,637 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:20,637 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:20,637 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:20,637 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:20,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48396 deadline: 1731857900634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:20,637 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:20,638 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:20,638 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:20,690 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-17T15:37:20,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. 2024-11-17T15:37:20,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. as already flushing 2024-11-17T15:37:20,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. 2024-11-17T15:37:20,690 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 34c72dd45fe3d9e6a1558a16ef53eb06, NAME => 'TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:37:20,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 34c72dd45fe3d9e6a1558a16ef53eb06, NAME => 'TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:37:20,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 34c72dd45fe3d9e6a1558a16ef53eb06, NAME => 'TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 34c72dd45fe3d9e6a1558a16ef53eb06, NAME => 'TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:37:20,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-17T15:37:20,844 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-17T15:37:20,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. 2024-11-17T15:37:20,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. as already flushing 2024-11-17T15:37:20,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. 2024-11-17T15:37:20,845 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 34c72dd45fe3d9e6a1558a16ef53eb06, NAME => 'TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:37:20,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 34c72dd45fe3d9e6a1558a16ef53eb06, NAME => 'TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:37:20,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 34c72dd45fe3d9e6a1558a16ef53eb06, NAME => 'TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 34c72dd45fe3d9e6a1558a16ef53eb06, NAME => 'TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:37:20,902 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/C/7f59a4ccad1543c4a2e67a41e77bce52 2024-11-17T15:37:20,913 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/A/42d8a488421545ed8912eb775fd9caaa as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/42d8a488421545ed8912eb775fd9caaa 2024-11-17T15:37:20,918 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/42d8a488421545ed8912eb775fd9caaa, entries=150, sequenceid=132, filesize=30.3 K 2024-11-17T15:37:20,920 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/B/4d02db3f5f28436580fac6958e3bb965 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/B/4d02db3f5f28436580fac6958e3bb965 2024-11-17T15:37:20,936 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/B/4d02db3f5f28436580fac6958e3bb965, entries=150, sequenceid=132, filesize=11.8 K 2024-11-17T15:37:20,937 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/C/7f59a4ccad1543c4a2e67a41e77bce52 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/C/7f59a4ccad1543c4a2e67a41e77bce52 2024-11-17T15:37:20,943 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/C/7f59a4ccad1543c4a2e67a41e77bce52, entries=150, sequenceid=132, filesize=11.8 K 2024-11-17T15:37:20,945 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 34c72dd45fe3d9e6a1558a16ef53eb06 in 1018ms, sequenceid=132, compaction requested=true 2024-11-17T15:37:20,945 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 34c72dd45fe3d9e6a1558a16ef53eb06: 2024-11-17T15:37:20,945 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 34c72dd45fe3d9e6a1558a16ef53eb06:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T15:37:20,945 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T15:37:20,945 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 34c72dd45fe3d9e6a1558a16ef53eb06:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T15:37:20,945 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T15:37:20,945 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 34c72dd45fe3d9e6a1558a16ef53eb06:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T15:37:20,945 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T15:37:20,945 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T15:37:20,945 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T15:37:20,950 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T15:37:20,950 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93171 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T15:37:20,950 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1541): 34c72dd45fe3d9e6a1558a16ef53eb06/B is initiating minor compaction (all files) 2024-11-17T15:37:20,950 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1541): 34c72dd45fe3d9e6a1558a16ef53eb06/A is initiating minor compaction (all files) 2024-11-17T15:37:20,950 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 34c72dd45fe3d9e6a1558a16ef53eb06/B in TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. 2024-11-17T15:37:20,950 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 34c72dd45fe3d9e6a1558a16ef53eb06/A in TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. 2024-11-17T15:37:20,950 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/B/35379b7f83784fdf90a10bce4fd4850c, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/B/ddf4281b810a4760a3dc29ab840b5225, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/B/4d02db3f5f28436580fac6958e3bb965] into tmpdir=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp, totalSize=35.5 K 2024-11-17T15:37:20,950 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/46801bd954344096976cf0e0ae332a6e, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/b6bfd52673314433b51ea810c706b8cf, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/42d8a488421545ed8912eb775fd9caaa] into tmpdir=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp, totalSize=91.0 K 2024-11-17T15:37:20,950 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. 2024-11-17T15:37:20,950 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. files: [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/46801bd954344096976cf0e0ae332a6e, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/b6bfd52673314433b51ea810c706b8cf, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/42d8a488421545ed8912eb775fd9caaa] 2024-11-17T15:37:20,950 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting 35379b7f83784fdf90a10bce4fd4850c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1731857837731 2024-11-17T15:37:20,951 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 46801bd954344096976cf0e0ae332a6e, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1731857837731 2024-11-17T15:37:20,951 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting ddf4281b810a4760a3dc29ab840b5225, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731857838101 2024-11-17T15:37:20,951 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting b6bfd52673314433b51ea810c706b8cf, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731857838101 2024-11-17T15:37:20,951 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting 4d02db3f5f28436580fac6958e3bb965, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731857839281 2024-11-17T15:37:20,951 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 42d8a488421545ed8912eb775fd9caaa, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731857839281 2024-11-17T15:37:20,966 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 34c72dd45fe3d9e6a1558a16ef53eb06#B#compaction#125 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T15:37:20,967 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/B/4a6775a1914b4a2daa785d91db2c2f3a is 50, key is test_row_0/B:col10/1731857839926/Put/seqid=0 2024-11-17T15:37:20,973 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=34c72dd45fe3d9e6a1558a16ef53eb06] 2024-11-17T15:37:20,979 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111725ef2576f1274ba4a96ccb68847c7185_34c72dd45fe3d9e6a1558a16ef53eb06 store=[table=TestAcidGuarantees family=A region=34c72dd45fe3d9e6a1558a16ef53eb06] 2024-11-17T15:37:20,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073741974_1150 (size=12409) 2024-11-17T15:37:20,983 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111725ef2576f1274ba4a96ccb68847c7185_34c72dd45fe3d9e6a1558a16ef53eb06, store=[table=TestAcidGuarantees family=A region=34c72dd45fe3d9e6a1558a16ef53eb06] 2024-11-17T15:37:20,983 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111725ef2576f1274ba4a96ccb68847c7185_34c72dd45fe3d9e6a1558a16ef53eb06 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=34c72dd45fe3d9e6a1558a16ef53eb06] 2024-11-17T15:37:20,998 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-17T15:37:20,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. 2024-11-17T15:37:20,998 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing 34c72dd45fe3d9e6a1558a16ef53eb06 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-17T15:37:21,000 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073741975_1151 (size=4469) 2024-11-17T15:37:21,003 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 34c72dd45fe3d9e6a1558a16ef53eb06#A#compaction#126 average throughput is 0.81 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T15:37:21,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-17T15:37:21,004 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/A/c23309e3ef4a41aa9d64ca27df001642 is 175, key is test_row_0/A:col10/1731857839926/Put/seqid=0 2024-11-17T15:37:21,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117ef2d17d9058548a690f179632236993a_34c72dd45fe3d9e6a1558a16ef53eb06 is 50, key is test_row_0/A:col10/1731857839951/Put/seqid=0 2024-11-17T15:37:21,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073741976_1152 (size=31363) 2024-11-17T15:37:21,055 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/A/c23309e3ef4a41aa9d64ca27df001642 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/c23309e3ef4a41aa9d64ca27df001642 2024-11-17T15:37:21,064 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 34c72dd45fe3d9e6a1558a16ef53eb06/A of 34c72dd45fe3d9e6a1558a16ef53eb06 into c23309e3ef4a41aa9d64ca27df001642(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T15:37:21,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073741977_1153 (size=12304) 2024-11-17T15:37:21,064 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 34c72dd45fe3d9e6a1558a16ef53eb06: 2024-11-17T15:37:21,064 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., storeName=34c72dd45fe3d9e6a1558a16ef53eb06/A, priority=13, startTime=1731857840945; duration=0sec 2024-11-17T15:37:21,065 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T15:37:21,065 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 34c72dd45fe3d9e6a1558a16ef53eb06:A 2024-11-17T15:37:21,065 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T15:37:21,066 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48242 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T15:37:21,066 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1541): 34c72dd45fe3d9e6a1558a16ef53eb06/C is initiating minor compaction (all files) 2024-11-17T15:37:21,066 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 34c72dd45fe3d9e6a1558a16ef53eb06/C in TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. 2024-11-17T15:37:21,067 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/C/f0d3442a16ba4f339f2f4bfb8141ba32, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/C/a8935481cbc74a389c98c34b0650c8d3, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/C/a792eefe4ce54c548f3fb56f254f0321, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/C/7f59a4ccad1543c4a2e67a41e77bce52] into tmpdir=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp, totalSize=47.1 K 2024-11-17T15:37:21,067 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting f0d3442a16ba4f339f2f4bfb8141ba32, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731857837085 2024-11-17T15:37:21,068 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting a8935481cbc74a389c98c34b0650c8d3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1731857837731 2024-11-17T15:37:21,068 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting a792eefe4ce54c548f3fb56f254f0321, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731857838101 2024-11-17T15:37:21,068 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7f59a4ccad1543c4a2e67a41e77bce52, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731857839281 2024-11-17T15:37:21,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:21,077 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117ef2d17d9058548a690f179632236993a_34c72dd45fe3d9e6a1558a16ef53eb06 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117ef2d17d9058548a690f179632236993a_34c72dd45fe3d9e6a1558a16ef53eb06 2024-11-17T15:37:21,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/A/9d4de9bdedda46688e210242dc598bda, store: [table=TestAcidGuarantees family=A region=34c72dd45fe3d9e6a1558a16ef53eb06] 2024-11-17T15:37:21,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/A/9d4de9bdedda46688e210242dc598bda is 175, key is test_row_0/A:col10/1731857839951/Put/seqid=0 2024-11-17T15:37:21,091 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 34c72dd45fe3d9e6a1558a16ef53eb06#C#compaction#128 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T15:37:21,092 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/C/6caead97ba93432eb1f4ab8ffaa756d4 is 50, key is test_row_0/C:col10/1731857839926/Put/seqid=0 2024-11-17T15:37:21,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073741978_1154 (size=31105) 2024-11-17T15:37:21,103 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=153, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/A/9d4de9bdedda46688e210242dc598bda 2024-11-17T15:37:21,118 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073741979_1155 (size=12375) 2024-11-17T15:37:21,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/B/40acabc06dd6406dbe403479ce937adb is 50, key is test_row_0/B:col10/1731857839951/Put/seqid=0 2024-11-17T15:37:21,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073741980_1156 (size=12151) 2024-11-17T15:37:21,135 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/B/40acabc06dd6406dbe403479ce937adb 2024-11-17T15:37:21,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(8855): Flush requested on 34c72dd45fe3d9e6a1558a16ef53eb06 2024-11-17T15:37:21,149 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. as already flushing 2024-11-17T15:37:21,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/C/adb4f2cd788b4cf085e9c464a8225a1d is 50, key is test_row_0/C:col10/1731857839951/Put/seqid=0 2024-11-17T15:37:21,163 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073741981_1157 (size=12151) 2024-11-17T15:37:21,164 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/C/adb4f2cd788b4cf085e9c464a8225a1d 2024-11-17T15:37:21,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/A/9d4de9bdedda46688e210242dc598bda as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/9d4de9bdedda46688e210242dc598bda 2024-11-17T15:37:21,173 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:21,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48428 deadline: 1731857901160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:21,174 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:21,174 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:21,174 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:21,174 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:21,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48420 deadline: 1731857901161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:21,174 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:21,174 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:21,174 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:21,174 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:21,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48396 deadline: 1731857901163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:21,175 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:21,175 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:21,175 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:21,178 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/9d4de9bdedda46688e210242dc598bda, entries=150, sequenceid=153, filesize=30.4 K 2024-11-17T15:37:21,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/B/40acabc06dd6406dbe403479ce937adb as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/B/40acabc06dd6406dbe403479ce937adb 2024-11-17T15:37:21,180 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:21,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48438 deadline: 1731857901173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:21,180 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:21,180 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:21,180 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:21,184 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/B/40acabc06dd6406dbe403479ce937adb, entries=150, sequenceid=153, filesize=11.9 K 2024-11-17T15:37:21,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/C/adb4f2cd788b4cf085e9c464a8225a1d as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/C/adb4f2cd788b4cf085e9c464a8225a1d 2024-11-17T15:37:21,189 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/C/adb4f2cd788b4cf085e9c464a8225a1d, entries=150, sequenceid=153, filesize=11.9 K 2024-11-17T15:37:21,190 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for 34c72dd45fe3d9e6a1558a16ef53eb06 in 192ms, sequenceid=153, compaction requested=false 2024-11-17T15:37:21,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for 34c72dd45fe3d9e6a1558a16ef53eb06: 2024-11-17T15:37:21,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. 2024-11-17T15:37:21,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-11-17T15:37:21,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-11-17T15:37:21,194 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-11-17T15:37:21,194 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 811 msec 2024-11-17T15:37:21,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(8855): Flush requested on 34c72dd45fe3d9e6a1558a16ef53eb06 2024-11-17T15:37:21,196 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T15:37:21,196 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 34c72dd45fe3d9e6a1558a16ef53eb06 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-17T15:37:21,196 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 821 msec 2024-11-17T15:37:21,209 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111793bb92ca7ee74f36ad8ff409d7766264_34c72dd45fe3d9e6a1558a16ef53eb06 is 50, key is test_row_0/A:col10/1731857841170/Put/seqid=0 2024-11-17T15:37:21,220 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073741982_1158 (size=14794) 2024-11-17T15:37:21,307 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:21,307 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:21,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48438 deadline: 1731857901294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:21,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48396 deadline: 1731857901294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:21,308 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:21,308 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:21,308 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:21,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48420 deadline: 1731857901294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:21,308 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:21,308 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:21,308 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:21,308 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:21,308 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:21,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48408 deadline: 1731857901296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:21,308 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:21,308 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:21,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48428 deadline: 1731857901296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:21,308 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:21,308 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:21,308 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:21,308 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:21,308 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:21,310 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:21,310 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:21,310 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:21,392 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/B/4a6775a1914b4a2daa785d91db2c2f3a as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/B/4a6775a1914b4a2daa785d91db2c2f3a 2024-11-17T15:37:21,398 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 34c72dd45fe3d9e6a1558a16ef53eb06/B of 34c72dd45fe3d9e6a1558a16ef53eb06 into 4a6775a1914b4a2daa785d91db2c2f3a(size=12.1 K), total size for store is 24.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T15:37:21,398 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 34c72dd45fe3d9e6a1558a16ef53eb06: 2024-11-17T15:37:21,398 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., storeName=34c72dd45fe3d9e6a1558a16ef53eb06/B, priority=13, startTime=1731857840945; duration=0sec 2024-11-17T15:37:21,398 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T15:37:21,398 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 34c72dd45fe3d9e6a1558a16ef53eb06:B 2024-11-17T15:37:21,420 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:21,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48438 deadline: 1731857901413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:21,420 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:21,420 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:21,420 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:21,420 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:21,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48420 deadline: 1731857901413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:21,421 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:21,421 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:21,421 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:21,421 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:21,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48396 deadline: 1731857901413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:21,421 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:21,421 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:21,421 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:21,421 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:21,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48428 deadline: 1731857901413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:21,422 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:21,422 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:21,422 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:21,422 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:21,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48408 deadline: 1731857901413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:21,422 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:21,422 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:21,422 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:21,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-17T15:37:21,514 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T15:37:21,514 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T15:37:21,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-11-17T15:37:21,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-17T15:37:21,516 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T15:37:21,517 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T15:37:21,517 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T15:37:21,526 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/C/6caead97ba93432eb1f4ab8ffaa756d4 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/C/6caead97ba93432eb1f4ab8ffaa756d4 2024-11-17T15:37:21,531 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 34c72dd45fe3d9e6a1558a16ef53eb06/C of 34c72dd45fe3d9e6a1558a16ef53eb06 into 6caead97ba93432eb1f4ab8ffaa756d4(size=12.1 K), total size for store is 24.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T15:37:21,531 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 34c72dd45fe3d9e6a1558a16ef53eb06: 2024-11-17T15:37:21,531 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., storeName=34c72dd45fe3d9e6a1558a16ef53eb06/C, priority=12, startTime=1731857840945; duration=0sec 2024-11-17T15:37:21,531 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T15:37:21,531 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 34c72dd45fe3d9e6a1558a16ef53eb06:C 2024-11-17T15:37:21,621 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:21,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-17T15:37:21,625 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:21,625 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:21,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48420 deadline: 1731857901624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:21,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48438 deadline: 1731857901624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:21,626 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:21,626 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:21,626 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:21,626 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:21,626 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:21,626 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:21,629 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111793bb92ca7ee74f36ad8ff409d7766264_34c72dd45fe3d9e6a1558a16ef53eb06 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111793bb92ca7ee74f36ad8ff409d7766264_34c72dd45fe3d9e6a1558a16ef53eb06 2024-11-17T15:37:21,630 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/A/8eb91122da1e47c983cd56f610f153f3, store: [table=TestAcidGuarantees family=A region=34c72dd45fe3d9e6a1558a16ef53eb06] 2024-11-17T15:37:21,631 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/A/8eb91122da1e47c983cd56f610f153f3 is 175, key is test_row_0/A:col10/1731857841170/Put/seqid=0 2024-11-17T15:37:21,637 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073741983_1159 (size=39749) 2024-11-17T15:37:21,637 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=170, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/A/8eb91122da1e47c983cd56f610f153f3 2024-11-17T15:37:21,637 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:21,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48396 deadline: 1731857901633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:21,638 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:21,638 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:21,638 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:21,638 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:21,638 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:21,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48408 deadline: 1731857901633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:21,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48428 deadline: 1731857901633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:21,639 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:21,639 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:21,639 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:21,640 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:21,640 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:21,640 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:21,648 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/B/e161e2d4072040a3a62538911f0d5c95 is 50, key is test_row_0/B:col10/1731857841170/Put/seqid=0 2024-11-17T15:37:21,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073741984_1160 (size=12151) 2024-11-17T15:37:21,670 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-17T15:37:21,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. 2024-11-17T15:37:21,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. as already flushing 2024-11-17T15:37:21,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. 2024-11-17T15:37:21,670 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 34c72dd45fe3d9e6a1558a16ef53eb06, NAME => 'TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:37:21,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 34c72dd45fe3d9e6a1558a16ef53eb06, NAME => 'TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:37:21,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 34c72dd45fe3d9e6a1558a16ef53eb06, NAME => 'TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 34c72dd45fe3d9e6a1558a16ef53eb06, NAME => 'TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:37:21,824 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-17T15:37:21,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. 2024-11-17T15:37:21,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. as already flushing 2024-11-17T15:37:21,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. 2024-11-17T15:37:21,824 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 34c72dd45fe3d9e6a1558a16ef53eb06, NAME => 'TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:37:21,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 34c72dd45fe3d9e6a1558a16ef53eb06, NAME => 'TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:37:21,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 34c72dd45fe3d9e6a1558a16ef53eb06, NAME => 'TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 34c72dd45fe3d9e6a1558a16ef53eb06, NAME => 'TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:37:21,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-17T15:37:21,935 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:21,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48420 deadline: 1731857901933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:21,936 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:21,936 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:21,936 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:21,936 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:21,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48438 deadline: 1731857901933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:21,937 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:21,937 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:21,937 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:21,950 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:21,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48396 deadline: 1731857901943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:21,950 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:21,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48428 deadline: 1731857901943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:21,951 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:21,951 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:21,951 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:21,951 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:21,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48408 deadline: 1731857901943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:21,951 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:21,951 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:21,952 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:21,952 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:21,952 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:21,952 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:21,977 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-17T15:37:21,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. 2024-11-17T15:37:21,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. as already flushing 2024-11-17T15:37:21,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. 2024-11-17T15:37:21,978 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 34c72dd45fe3d9e6a1558a16ef53eb06, NAME => 'TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:37:21,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 34c72dd45fe3d9e6a1558a16ef53eb06, NAME => 'TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:37:21,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 34c72dd45fe3d9e6a1558a16ef53eb06, NAME => 'TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 34c72dd45fe3d9e6a1558a16ef53eb06, NAME => 'TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:37:22,059 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/B/e161e2d4072040a3a62538911f0d5c95 2024-11-17T15:37:22,068 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/C/d5e9c4855b0d4fe8aed377e03a8972e2 is 50, key is test_row_0/C:col10/1731857841170/Put/seqid=0 2024-11-17T15:37:22,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073741985_1161 (size=12151) 2024-11-17T15:37:22,129 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-17T15:37:22,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. 2024-11-17T15:37:22,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. as already flushing 2024-11-17T15:37:22,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. 2024-11-17T15:37:22,130 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 34c72dd45fe3d9e6a1558a16ef53eb06, NAME => 'TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:37:22,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 34c72dd45fe3d9e6a1558a16ef53eb06, NAME => 'TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:37:22,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 34c72dd45fe3d9e6a1558a16ef53eb06, NAME => 'TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 34c72dd45fe3d9e6a1558a16ef53eb06, NAME => 'TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:37:22,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-17T15:37:22,283 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-17T15:37:22,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. 2024-11-17T15:37:22,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. as already flushing 2024-11-17T15:37:22,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. 2024-11-17T15:37:22,284 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 34c72dd45fe3d9e6a1558a16ef53eb06, NAME => 'TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:37:22,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 34c72dd45fe3d9e6a1558a16ef53eb06, NAME => 'TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:37:22,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 34c72dd45fe3d9e6a1558a16ef53eb06, NAME => 'TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 34c72dd45fe3d9e6a1558a16ef53eb06, NAME => 'TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:37:22,438 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-17T15:37:22,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. 2024-11-17T15:37:22,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. as already flushing 2024-11-17T15:37:22,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. 2024-11-17T15:37:22,438 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 34c72dd45fe3d9e6a1558a16ef53eb06, NAME => 'TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:37:22,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 34c72dd45fe3d9e6a1558a16ef53eb06, NAME => 'TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:37:22,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 34c72dd45fe3d9e6a1558a16ef53eb06, NAME => 'TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 34c72dd45fe3d9e6a1558a16ef53eb06, NAME => 'TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:37:22,446 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:22,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48420 deadline: 1731857902443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:22,447 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:22,447 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:22,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48438 deadline: 1731857902444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:22,447 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:22,447 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:22,447 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:22,447 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:22,447 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:22,464 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:22,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48428 deadline: 1731857902463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:22,464 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:22,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48396 deadline: 1731857902463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:22,465 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:22,465 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:22,465 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:22,465 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:22,465 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:22,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48408 deadline: 1731857902463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:22,465 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:22,465 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:22,465 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:22,465 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:22,465 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:22,490 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/C/d5e9c4855b0d4fe8aed377e03a8972e2 2024-11-17T15:37:22,496 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/A/8eb91122da1e47c983cd56f610f153f3 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/8eb91122da1e47c983cd56f610f153f3 2024-11-17T15:37:22,500 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/8eb91122da1e47c983cd56f610f153f3, entries=200, sequenceid=170, filesize=38.8 K 2024-11-17T15:37:22,501 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/B/e161e2d4072040a3a62538911f0d5c95 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/B/e161e2d4072040a3a62538911f0d5c95 2024-11-17T15:37:22,505 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/B/e161e2d4072040a3a62538911f0d5c95, entries=150, sequenceid=170, filesize=11.9 K 2024-11-17T15:37:22,506 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/C/d5e9c4855b0d4fe8aed377e03a8972e2 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/C/d5e9c4855b0d4fe8aed377e03a8972e2 2024-11-17T15:37:22,510 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/C/d5e9c4855b0d4fe8aed377e03a8972e2, entries=150, sequenceid=170, filesize=11.9 K 2024-11-17T15:37:22,511 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=120.76 KB/123660 for 34c72dd45fe3d9e6a1558a16ef53eb06 in 1315ms, sequenceid=170, compaction requested=true 2024-11-17T15:37:22,511 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 34c72dd45fe3d9e6a1558a16ef53eb06: 2024-11-17T15:37:22,511 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 34c72dd45fe3d9e6a1558a16ef53eb06:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T15:37:22,511 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T15:37:22,512 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 34c72dd45fe3d9e6a1558a16ef53eb06:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T15:37:22,512 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T15:37:22,512 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T15:37:22,512 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T15:37:22,512 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 34c72dd45fe3d9e6a1558a16ef53eb06:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T15:37:22,512 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T15:37:22,513 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102217 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T15:37:22,513 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T15:37:22,513 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1541): 34c72dd45fe3d9e6a1558a16ef53eb06/A is initiating minor compaction (all files) 2024-11-17T15:37:22,513 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1541): 34c72dd45fe3d9e6a1558a16ef53eb06/B is initiating minor compaction (all files) 2024-11-17T15:37:22,513 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 34c72dd45fe3d9e6a1558a16ef53eb06/A in TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. 2024-11-17T15:37:22,513 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 34c72dd45fe3d9e6a1558a16ef53eb06/B in TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. 2024-11-17T15:37:22,513 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/c23309e3ef4a41aa9d64ca27df001642, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/9d4de9bdedda46688e210242dc598bda, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/8eb91122da1e47c983cd56f610f153f3] into tmpdir=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp, totalSize=99.8 K 2024-11-17T15:37:22,513 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/B/4a6775a1914b4a2daa785d91db2c2f3a, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/B/40acabc06dd6406dbe403479ce937adb, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/B/e161e2d4072040a3a62538911f0d5c95] into tmpdir=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp, totalSize=35.9 K 2024-11-17T15:37:22,513 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. 2024-11-17T15:37:22,513 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. files: [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/c23309e3ef4a41aa9d64ca27df001642, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/9d4de9bdedda46688e210242dc598bda, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/8eb91122da1e47c983cd56f610f153f3] 2024-11-17T15:37:22,513 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting 4a6775a1914b4a2daa785d91db2c2f3a, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731857839281 2024-11-17T15:37:22,513 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting c23309e3ef4a41aa9d64ca27df001642, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731857839281 2024-11-17T15:37:22,514 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9d4de9bdedda46688e210242dc598bda, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1731857839951 2024-11-17T15:37:22,514 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting 40acabc06dd6406dbe403479ce937adb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1731857839951 2024-11-17T15:37:22,514 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8eb91122da1e47c983cd56f610f153f3, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731857841157 2024-11-17T15:37:22,515 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting e161e2d4072040a3a62538911f0d5c95, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731857841157 2024-11-17T15:37:22,523 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=34c72dd45fe3d9e6a1558a16ef53eb06] 2024-11-17T15:37:22,525 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 34c72dd45fe3d9e6a1558a16ef53eb06#B#compaction#135 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T15:37:22,526 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/B/558f2d07243c42f6802aac8dae181762 is 50, key is test_row_0/B:col10/1731857841170/Put/seqid=0 2024-11-17T15:37:22,527 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411171f90f27ec6234eaa9fd803fbc8872e87_34c72dd45fe3d9e6a1558a16ef53eb06 store=[table=TestAcidGuarantees family=A region=34c72dd45fe3d9e6a1558a16ef53eb06] 2024-11-17T15:37:22,529 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411171f90f27ec6234eaa9fd803fbc8872e87_34c72dd45fe3d9e6a1558a16ef53eb06, store=[table=TestAcidGuarantees family=A region=34c72dd45fe3d9e6a1558a16ef53eb06] 2024-11-17T15:37:22,529 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411171f90f27ec6234eaa9fd803fbc8872e87_34c72dd45fe3d9e6a1558a16ef53eb06 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=34c72dd45fe3d9e6a1558a16ef53eb06] 2024-11-17T15:37:22,547 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073741986_1162 (size=12561) 2024-11-17T15:37:22,556 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073741987_1163 (size=4469) 2024-11-17T15:37:22,556 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/B/558f2d07243c42f6802aac8dae181762 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/B/558f2d07243c42f6802aac8dae181762 2024-11-17T15:37:22,558 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 34c72dd45fe3d9e6a1558a16ef53eb06#A#compaction#134 average throughput is 0.70 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T15:37:22,558 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/A/1b9f26eb6f114219be2a34a39a2bec9f is 175, key is test_row_0/A:col10/1731857841170/Put/seqid=0 2024-11-17T15:37:22,565 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 34c72dd45fe3d9e6a1558a16ef53eb06/B of 34c72dd45fe3d9e6a1558a16ef53eb06 into 558f2d07243c42f6802aac8dae181762(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T15:37:22,565 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 34c72dd45fe3d9e6a1558a16ef53eb06: 2024-11-17T15:37:22,565 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., storeName=34c72dd45fe3d9e6a1558a16ef53eb06/B, priority=13, startTime=1731857842511; duration=0sec 2024-11-17T15:37:22,565 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T15:37:22,565 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 34c72dd45fe3d9e6a1558a16ef53eb06:B 2024-11-17T15:37:22,565 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T15:37:22,566 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36677 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T15:37:22,567 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1541): 34c72dd45fe3d9e6a1558a16ef53eb06/C is initiating minor compaction (all files) 2024-11-17T15:37:22,567 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 34c72dd45fe3d9e6a1558a16ef53eb06/C in TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. 2024-11-17T15:37:22,567 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/C/6caead97ba93432eb1f4ab8ffaa756d4, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/C/adb4f2cd788b4cf085e9c464a8225a1d, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/C/d5e9c4855b0d4fe8aed377e03a8972e2] into tmpdir=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp, totalSize=35.8 K 2024-11-17T15:37:22,568 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting 6caead97ba93432eb1f4ab8ffaa756d4, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731857839281 2024-11-17T15:37:22,569 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting adb4f2cd788b4cf085e9c464a8225a1d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1731857839951 2024-11-17T15:37:22,570 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting d5e9c4855b0d4fe8aed377e03a8972e2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731857841157 2024-11-17T15:37:22,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073741988_1164 (size=31515) 2024-11-17T15:37:22,580 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 34c72dd45fe3d9e6a1558a16ef53eb06#C#compaction#136 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T15:37:22,580 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/C/3821e93ba88246ad9dc47e96bc7763dd is 50, key is test_row_0/C:col10/1731857841170/Put/seqid=0 2024-11-17T15:37:22,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073741989_1165 (size=12527) 2024-11-17T15:37:22,592 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-17T15:37:22,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. 2024-11-17T15:37:22,592 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing 34c72dd45fe3d9e6a1558a16ef53eb06 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-17T15:37:22,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117f3bc7e29ee9d48518f1015ddb65342f8_34c72dd45fe3d9e6a1558a16ef53eb06 is 50, key is test_row_0/A:col10/1731857841287/Put/seqid=0 2024-11-17T15:37:22,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073741990_1166 (size=12304) 2024-11-17T15:37:22,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-17T15:37:22,982 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/A/1b9f26eb6f114219be2a34a39a2bec9f as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/1b9f26eb6f114219be2a34a39a2bec9f 2024-11-17T15:37:22,993 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/C/3821e93ba88246ad9dc47e96bc7763dd as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/C/3821e93ba88246ad9dc47e96bc7763dd 2024-11-17T15:37:22,994 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 34c72dd45fe3d9e6a1558a16ef53eb06/A of 34c72dd45fe3d9e6a1558a16ef53eb06 into 1b9f26eb6f114219be2a34a39a2bec9f(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T15:37:22,994 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 34c72dd45fe3d9e6a1558a16ef53eb06: 2024-11-17T15:37:22,995 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., storeName=34c72dd45fe3d9e6a1558a16ef53eb06/A, priority=13, startTime=1731857842511; duration=0sec 2024-11-17T15:37:22,995 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T15:37:22,995 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 34c72dd45fe3d9e6a1558a16ef53eb06:A 2024-11-17T15:37:23,000 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 34c72dd45fe3d9e6a1558a16ef53eb06/C of 34c72dd45fe3d9e6a1558a16ef53eb06 into 3821e93ba88246ad9dc47e96bc7763dd(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T15:37:23,000 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 34c72dd45fe3d9e6a1558a16ef53eb06: 2024-11-17T15:37:23,000 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., storeName=34c72dd45fe3d9e6a1558a16ef53eb06/C, priority=13, startTime=1731857842512; duration=0sec 2024-11-17T15:37:23,000 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T15:37:23,000 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 34c72dd45fe3d9e6a1558a16ef53eb06:C 2024-11-17T15:37:23,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:23,042 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117f3bc7e29ee9d48518f1015ddb65342f8_34c72dd45fe3d9e6a1558a16ef53eb06 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117f3bc7e29ee9d48518f1015ddb65342f8_34c72dd45fe3d9e6a1558a16ef53eb06 2024-11-17T15:37:23,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/A/1f6ba0fbf2ec4117858a604d48428b0e, store: [table=TestAcidGuarantees family=A region=34c72dd45fe3d9e6a1558a16ef53eb06] 2024-11-17T15:37:23,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/A/1f6ba0fbf2ec4117858a604d48428b0e is 175, key is test_row_0/A:col10/1731857841287/Put/seqid=0 2024-11-17T15:37:23,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073741991_1167 (size=31105) 2024-11-17T15:37:23,053 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=194, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/A/1f6ba0fbf2ec4117858a604d48428b0e 2024-11-17T15:37:23,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/B/a2cc53bf68d54bb39edd62cc83d10932 is 50, key is test_row_0/B:col10/1731857841287/Put/seqid=0 2024-11-17T15:37:23,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073741992_1168 (size=12151) 2024-11-17T15:37:23,086 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/B/a2cc53bf68d54bb39edd62cc83d10932 2024-11-17T15:37:23,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/C/b8f35c4400a44600b787d03d95d309d5 is 50, key is test_row_0/C:col10/1731857841287/Put/seqid=0 2024-11-17T15:37:23,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073741993_1169 (size=12151) 2024-11-17T15:37:23,457 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. as already flushing 2024-11-17T15:37:23,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(8855): Flush requested on 34c72dd45fe3d9e6a1558a16ef53eb06 2024-11-17T15:37:23,482 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:23,482 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:23,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48396 deadline: 1731857903473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:23,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48438 deadline: 1731857903475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:23,483 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:23,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48420 deadline: 1731857903480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:23,483 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:23,483 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:23,483 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:23,483 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:23,483 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:23,483 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:23,483 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 7a780d55532c,32955,1731857800648 for 'test_row_0' in 34c72dd45fe3d9e6a1558a16ef53eb06 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2188 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T15:37:23,484 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:23,484 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:23,484 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:23,485 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:23,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48428 deadline: 1731857903482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:23,485 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:23,485 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:23,485 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:23,485 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:23,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48408 deadline: 1731857903483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:23,486 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:23,486 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:23,486 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:23,510 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/C/b8f35c4400a44600b787d03d95d309d5 2024-11-17T15:37:23,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/A/1f6ba0fbf2ec4117858a604d48428b0e as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/1f6ba0fbf2ec4117858a604d48428b0e 2024-11-17T15:37:23,524 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/1f6ba0fbf2ec4117858a604d48428b0e, entries=150, sequenceid=194, filesize=30.4 K 2024-11-17T15:37:23,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/B/a2cc53bf68d54bb39edd62cc83d10932 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/B/a2cc53bf68d54bb39edd62cc83d10932 2024-11-17T15:37:23,530 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/B/a2cc53bf68d54bb39edd62cc83d10932, entries=150, sequenceid=194, filesize=11.9 K 2024-11-17T15:37:23,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/C/b8f35c4400a44600b787d03d95d309d5 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/C/b8f35c4400a44600b787d03d95d309d5 2024-11-17T15:37:23,536 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/C/b8f35c4400a44600b787d03d95d309d5, entries=150, sequenceid=194, filesize=11.9 K 2024-11-17T15:37:23,538 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=87.22 KB/89310 for 34c72dd45fe3d9e6a1558a16ef53eb06 in 946ms, sequenceid=194, compaction requested=false 2024-11-17T15:37:23,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for 34c72dd45fe3d9e6a1558a16ef53eb06: 2024-11-17T15:37:23,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. 2024-11-17T15:37:23,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-11-17T15:37:23,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-11-17T15:37:23,543 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-11-17T15:37:23,543 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0230 sec 2024-11-17T15:37:23,546 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 2.0300 sec 2024-11-17T15:37:23,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(8855): Flush requested on 34c72dd45fe3d9e6a1558a16ef53eb06 2024-11-17T15:37:23,597 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T15:37:23,597 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 34c72dd45fe3d9e6a1558a16ef53eb06 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-17T15:37:23,607 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117fd9629689cac4af480f7869ab06c983d_34c72dd45fe3d9e6a1558a16ef53eb06 is 50, key is test_row_0/A:col10/1731857843597/Put/seqid=0 2024-11-17T15:37:23,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073741994_1170 (size=17284) 2024-11-17T15:37:23,613 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:23,623 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117fd9629689cac4af480f7869ab06c983d_34c72dd45fe3d9e6a1558a16ef53eb06 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117fd9629689cac4af480f7869ab06c983d_34c72dd45fe3d9e6a1558a16ef53eb06 2024-11-17T15:37:23,624 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:23,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48408 deadline: 1731857903616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:23,624 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/A/92c3f89127634f18981173509e4f423c, store: [table=TestAcidGuarantees family=A region=34c72dd45fe3d9e6a1558a16ef53eb06] 2024-11-17T15:37:23,624 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:23,624 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:23,625 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:23,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48438 deadline: 1731857903617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:23,625 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:23,625 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/A/92c3f89127634f18981173509e4f423c is 175, key is test_row_0/A:col10/1731857843597/Put/seqid=0 2024-11-17T15:37:23,625 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:23,625 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:23,625 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:23,625 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:23,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48420 deadline: 1731857903618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:23,625 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:23,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48428 deadline: 1731857903619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:23,626 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:23,626 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:23,626 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:23,626 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:23,626 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:23,626 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:23,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073741995_1171 (size=48389) 2024-11-17T15:37:23,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-17T15:37:23,664 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T15:37:23,664 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T15:37:23,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees 2024-11-17T15:37:23,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-17T15:37:23,666 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T15:37:23,667 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T15:37:23,667 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T15:37:23,734 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:23,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48408 deadline: 1731857903733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:23,734 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:23,735 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:23,735 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:23,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48438 deadline: 1731857903733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:23,735 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:23,735 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:23,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48428 deadline: 1731857903733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:23,735 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:23,735 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:23,735 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:23,735 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:23,735 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:23,735 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:23,735 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:23,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48420 deadline: 1731857903733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:23,736 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:23,736 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:23,736 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:23,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-17T15:37:23,820 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-17T15:37:23,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. 2024-11-17T15:37:23,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. as already flushing 2024-11-17T15:37:23,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. 2024-11-17T15:37:23,820 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 34c72dd45fe3d9e6a1558a16ef53eb06, NAME => 'TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:37:23,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 34c72dd45fe3d9e6a1558a16ef53eb06, NAME => 'TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:37:23,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 34c72dd45fe3d9e6a1558a16ef53eb06, NAME => 'TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 34c72dd45fe3d9e6a1558a16ef53eb06, NAME => 'TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:37:23,944 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:23,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48408 deadline: 1731857903943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:23,945 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:23,945 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:23,945 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:23,945 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:23,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48438 deadline: 1731857903943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:23,945 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:23,945 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:23,945 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:23,945 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:23,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48428 deadline: 1731857903943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:23,945 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:23,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48420 deadline: 1731857903943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:23,946 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:23,946 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:23,946 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:23,946 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:23,946 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:23,946 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:23,973 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-17T15:37:23,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. 2024-11-17T15:37:23,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. as already flushing 2024-11-17T15:37:23,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. 2024-11-17T15:37:23,974 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 34c72dd45fe3d9e6a1558a16ef53eb06, NAME => 'TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:37:23,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 34c72dd45fe3d9e6a1558a16ef53eb06, NAME => 'TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:37:23,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 34c72dd45fe3d9e6a1558a16ef53eb06, NAME => 'TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 34c72dd45fe3d9e6a1558a16ef53eb06, NAME => 'TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:37:23,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-17T15:37:24,031 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=216, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/A/92c3f89127634f18981173509e4f423c 2024-11-17T15:37:24,041 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/B/8a19ca1080d5476e9431e84401a2dc33 is 50, key is test_row_0/B:col10/1731857843597/Put/seqid=0 2024-11-17T15:37:24,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073741996_1172 (size=12151) 2024-11-17T15:37:24,047 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=216 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/B/8a19ca1080d5476e9431e84401a2dc33 2024-11-17T15:37:24,056 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/C/a84beb7952024cc38e7c159a689a0864 is 50, key is test_row_0/C:col10/1731857843597/Put/seqid=0 2024-11-17T15:37:24,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073741997_1173 (size=12151) 2024-11-17T15:37:24,062 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=216 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/C/a84beb7952024cc38e7c159a689a0864 2024-11-17T15:37:24,068 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/A/92c3f89127634f18981173509e4f423c as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/92c3f89127634f18981173509e4f423c 2024-11-17T15:37:24,072 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/92c3f89127634f18981173509e4f423c, entries=250, sequenceid=216, filesize=47.3 K 2024-11-17T15:37:24,073 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/B/8a19ca1080d5476e9431e84401a2dc33 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/B/8a19ca1080d5476e9431e84401a2dc33 2024-11-17T15:37:24,078 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/B/8a19ca1080d5476e9431e84401a2dc33, entries=150, sequenceid=216, filesize=11.9 K 2024-11-17T15:37:24,079 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/C/a84beb7952024cc38e7c159a689a0864 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/C/a84beb7952024cc38e7c159a689a0864 2024-11-17T15:37:24,085 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/C/a84beb7952024cc38e7c159a689a0864, entries=150, sequenceid=216, filesize=11.9 K 2024-11-17T15:37:24,086 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for 34c72dd45fe3d9e6a1558a16ef53eb06 in 489ms, sequenceid=216, compaction requested=true 2024-11-17T15:37:24,086 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 34c72dd45fe3d9e6a1558a16ef53eb06: 2024-11-17T15:37:24,086 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 34c72dd45fe3d9e6a1558a16ef53eb06:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T15:37:24,086 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T15:37:24,086 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 34c72dd45fe3d9e6a1558a16ef53eb06:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T15:37:24,086 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T15:37:24,086 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 34c72dd45fe3d9e6a1558a16ef53eb06:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T15:37:24,086 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T15:37:24,086 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T15:37:24,086 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T15:37:24,087 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111009 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T15:37:24,088 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1541): 34c72dd45fe3d9e6a1558a16ef53eb06/A is initiating minor compaction (all files) 2024-11-17T15:37:24,088 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 34c72dd45fe3d9e6a1558a16ef53eb06/A in TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. 2024-11-17T15:37:24,088 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/1b9f26eb6f114219be2a34a39a2bec9f, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/1f6ba0fbf2ec4117858a604d48428b0e, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/92c3f89127634f18981173509e4f423c] into tmpdir=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp, totalSize=108.4 K 2024-11-17T15:37:24,088 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. 2024-11-17T15:37:24,088 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. files: [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/1b9f26eb6f114219be2a34a39a2bec9f, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/1f6ba0fbf2ec4117858a604d48428b0e, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/92c3f89127634f18981173509e4f423c] 2024-11-17T15:37:24,088 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T15:37:24,088 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1541): 34c72dd45fe3d9e6a1558a16ef53eb06/B is initiating minor compaction (all files) 2024-11-17T15:37:24,088 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 34c72dd45fe3d9e6a1558a16ef53eb06/B in TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. 2024-11-17T15:37:24,088 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/B/558f2d07243c42f6802aac8dae181762, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/B/a2cc53bf68d54bb39edd62cc83d10932, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/B/8a19ca1080d5476e9431e84401a2dc33] into tmpdir=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp, totalSize=36.0 K 2024-11-17T15:37:24,090 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1b9f26eb6f114219be2a34a39a2bec9f, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731857841157 2024-11-17T15:37:24,090 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting 558f2d07243c42f6802aac8dae181762, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731857841157 2024-11-17T15:37:24,090 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1f6ba0fbf2ec4117858a604d48428b0e, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1731857841287 2024-11-17T15:37:24,090 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting a2cc53bf68d54bb39edd62cc83d10932, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1731857841287 2024-11-17T15:37:24,091 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 92c3f89127634f18981173509e4f423c, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1731857843468 2024-11-17T15:37:24,091 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting 8a19ca1080d5476e9431e84401a2dc33, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1731857843596 2024-11-17T15:37:24,101 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=34c72dd45fe3d9e6a1558a16ef53eb06] 2024-11-17T15:37:24,106 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 34c72dd45fe3d9e6a1558a16ef53eb06#B#compaction#144 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T15:37:24,106 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/B/af2c4bc77ca34aae960590fc332a2a0f is 50, key is test_row_0/B:col10/1731857843597/Put/seqid=0 2024-11-17T15:37:24,116 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241117ca45f8c60e2140489f0c6c49ae537a49_34c72dd45fe3d9e6a1558a16ef53eb06 store=[table=TestAcidGuarantees family=A region=34c72dd45fe3d9e6a1558a16ef53eb06] 2024-11-17T15:37:24,117 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241117ca45f8c60e2140489f0c6c49ae537a49_34c72dd45fe3d9e6a1558a16ef53eb06, store=[table=TestAcidGuarantees family=A region=34c72dd45fe3d9e6a1558a16ef53eb06] 2024-11-17T15:37:24,118 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117ca45f8c60e2140489f0c6c49ae537a49_34c72dd45fe3d9e6a1558a16ef53eb06 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=34c72dd45fe3d9e6a1558a16ef53eb06] 2024-11-17T15:37:24,127 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-17T15:37:24,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. 2024-11-17T15:37:24,127 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2902): Flushing 34c72dd45fe3d9e6a1558a16ef53eb06 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-17T15:37:24,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073741998_1174 (size=12663) 2024-11-17T15:37:24,143 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/B/af2c4bc77ca34aae960590fc332a2a0f as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/B/af2c4bc77ca34aae960590fc332a2a0f 2024-11-17T15:37:24,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411171465ac8db9ef4f9784468a5b621ed099_34c72dd45fe3d9e6a1558a16ef53eb06 is 50, key is test_row_0/A:col10/1731857843614/Put/seqid=0 2024-11-17T15:37:24,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073741999_1175 (size=4469) 2024-11-17T15:37:24,155 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 34c72dd45fe3d9e6a1558a16ef53eb06/B of 34c72dd45fe3d9e6a1558a16ef53eb06 into af2c4bc77ca34aae960590fc332a2a0f(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T15:37:24,155 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 34c72dd45fe3d9e6a1558a16ef53eb06: 2024-11-17T15:37:24,155 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., storeName=34c72dd45fe3d9e6a1558a16ef53eb06/B, priority=13, startTime=1731857844086; duration=0sec 2024-11-17T15:37:24,155 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T15:37:24,156 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 34c72dd45fe3d9e6a1558a16ef53eb06:B 2024-11-17T15:37:24,156 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T15:37:24,157 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T15:37:24,157 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1541): 34c72dd45fe3d9e6a1558a16ef53eb06/C is initiating minor compaction (all files) 2024-11-17T15:37:24,157 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 34c72dd45fe3d9e6a1558a16ef53eb06/C in TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. 2024-11-17T15:37:24,158 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/C/3821e93ba88246ad9dc47e96bc7763dd, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/C/b8f35c4400a44600b787d03d95d309d5, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/C/a84beb7952024cc38e7c159a689a0864] into tmpdir=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp, totalSize=36.0 K 2024-11-17T15:37:24,158 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting 3821e93ba88246ad9dc47e96bc7763dd, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731857841157 2024-11-17T15:37:24,159 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting b8f35c4400a44600b787d03d95d309d5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1731857841287 2024-11-17T15:37:24,159 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting a84beb7952024cc38e7c159a689a0864, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1731857843596 2024-11-17T15:37:24,169 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 34c72dd45fe3d9e6a1558a16ef53eb06#C#compaction#146 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T15:37:24,170 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/C/99c73440b67a416d9cf345e797bcda0a is 50, key is test_row_0/C:col10/1731857843597/Put/seqid=0 2024-11-17T15:37:24,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742000_1176 (size=12304) 2024-11-17T15:37:24,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:24,176 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411171465ac8db9ef4f9784468a5b621ed099_34c72dd45fe3d9e6a1558a16ef53eb06 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411171465ac8db9ef4f9784468a5b621ed099_34c72dd45fe3d9e6a1558a16ef53eb06 2024-11-17T15:37:24,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/A/308900f4496248a285d0daabb50408cd, store: [table=TestAcidGuarantees family=A region=34c72dd45fe3d9e6a1558a16ef53eb06] 2024-11-17T15:37:24,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/A/308900f4496248a285d0daabb50408cd is 175, key is test_row_0/A:col10/1731857843614/Put/seqid=0 2024-11-17T15:37:24,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742001_1177 (size=12629) 2024-11-17T15:37:24,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742002_1178 (size=31105) 2024-11-17T15:37:24,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(8855): Flush requested on 34c72dd45fe3d9e6a1558a16ef53eb06 2024-11-17T15:37:24,255 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. as already flushing 2024-11-17T15:37:24,290 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:24,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48438 deadline: 1731857904283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:24,291 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:24,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48420 deadline: 1731857904284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:24,291 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:24,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48408 deadline: 1731857904286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:24,291 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:24,292 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:24,292 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:24,292 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:24,292 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:24,292 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:24,292 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:24,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48428 deadline: 1731857904288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:24,292 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:24,292 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:24,292 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:24,292 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:24,292 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:24,292 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:24,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-17T15:37:24,394 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:24,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48438 deadline: 1731857904393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:24,395 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:24,395 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:24,395 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:24,406 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:24,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48420 deadline: 1731857904403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:24,406 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:24,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48428 deadline: 1731857904403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:24,406 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:24,406 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:24,406 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:24,406 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:24,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48408 deadline: 1731857904403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:24,407 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:24,407 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:24,407 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:24,407 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:24,407 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:24,407 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:24,553 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 34c72dd45fe3d9e6a1558a16ef53eb06#A#compaction#143 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T15:37:24,554 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/A/819bfa332f2549c2af5a739a8fad5fdf is 175, key is test_row_0/A:col10/1731857843597/Put/seqid=0 2024-11-17T15:37:24,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742003_1179 (size=31617) 2024-11-17T15:37:24,591 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/C/99c73440b67a416d9cf345e797bcda0a as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/C/99c73440b67a416d9cf345e797bcda0a 2024-11-17T15:37:24,596 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 34c72dd45fe3d9e6a1558a16ef53eb06/C of 34c72dd45fe3d9e6a1558a16ef53eb06 into 99c73440b67a416d9cf345e797bcda0a(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T15:37:24,596 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 34c72dd45fe3d9e6a1558a16ef53eb06: 2024-11-17T15:37:24,596 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., storeName=34c72dd45fe3d9e6a1558a16ef53eb06/C, priority=13, startTime=1731857844086; duration=0sec 2024-11-17T15:37:24,597 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T15:37:24,597 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 34c72dd45fe3d9e6a1558a16ef53eb06:C 2024-11-17T15:37:24,598 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=232, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/A/308900f4496248a285d0daabb50408cd 2024-11-17T15:37:24,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/B/6f8dbf9aa98d41fc9bf4c3a736f65eb4 is 50, key is test_row_0/B:col10/1731857843614/Put/seqid=0 2024-11-17T15:37:24,605 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:24,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48438 deadline: 1731857904603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:24,605 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:24,605 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:24,605 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:24,608 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742004_1180 (size=12151) 2024-11-17T15:37:24,612 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=232 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/B/6f8dbf9aa98d41fc9bf4c3a736f65eb4 2024-11-17T15:37:24,614 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:24,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48420 deadline: 1731857904613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:24,615 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:24,615 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:24,615 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:24,615 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:24,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48408 deadline: 1731857904613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:24,615 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:24,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48428 deadline: 1731857904613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:24,615 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:24,615 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:24,615 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:24,615 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:24,616 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:24,616 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:24,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/C/72d4f83ff53e4106ac571c9507fa64fb is 50, key is test_row_0/C:col10/1731857843614/Put/seqid=0 2024-11-17T15:37:24,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742005_1181 (size=12151) 2024-11-17T15:37:24,674 INFO [master/7a780d55532c:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-17T15:37:24,674 INFO [master/7a780d55532c:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-17T15:37:24,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-17T15:37:24,915 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:24,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48438 deadline: 1731857904913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:24,915 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:24,915 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:24,915 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:24,926 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:24,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48420 deadline: 1731857904923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:24,926 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:24,926 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:24,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48408 deadline: 1731857904923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:24,926 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:24,926 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:24,926 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:24,926 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:24,926 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:24,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48428 deadline: 1731857904923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:24,926 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:24,927 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:24,927 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:24,927 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:24,964 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/A/819bfa332f2549c2af5a739a8fad5fdf as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/819bfa332f2549c2af5a739a8fad5fdf 2024-11-17T15:37:24,969 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 34c72dd45fe3d9e6a1558a16ef53eb06/A of 34c72dd45fe3d9e6a1558a16ef53eb06 into 819bfa332f2549c2af5a739a8fad5fdf(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T15:37:24,969 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 34c72dd45fe3d9e6a1558a16ef53eb06: 2024-11-17T15:37:24,969 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., storeName=34c72dd45fe3d9e6a1558a16ef53eb06/A, priority=13, startTime=1731857844086; duration=0sec 2024-11-17T15:37:24,969 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T15:37:24,969 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 34c72dd45fe3d9e6a1558a16ef53eb06:A 2024-11-17T15:37:25,025 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=232 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/C/72d4f83ff53e4106ac571c9507fa64fb 2024-11-17T15:37:25,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/A/308900f4496248a285d0daabb50408cd as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/308900f4496248a285d0daabb50408cd 2024-11-17T15:37:25,034 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/308900f4496248a285d0daabb50408cd, entries=150, sequenceid=232, filesize=30.4 K 2024-11-17T15:37:25,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/B/6f8dbf9aa98d41fc9bf4c3a736f65eb4 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/B/6f8dbf9aa98d41fc9bf4c3a736f65eb4 2024-11-17T15:37:25,040 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/B/6f8dbf9aa98d41fc9bf4c3a736f65eb4, entries=150, sequenceid=232, filesize=11.9 K 2024-11-17T15:37:25,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/C/72d4f83ff53e4106ac571c9507fa64fb as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/C/72d4f83ff53e4106ac571c9507fa64fb 2024-11-17T15:37:25,045 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/C/72d4f83ff53e4106ac571c9507fa64fb, entries=150, sequenceid=232, filesize=11.9 K 2024-11-17T15:37:25,046 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for 34c72dd45fe3d9e6a1558a16ef53eb06 in 919ms, sequenceid=232, compaction requested=false 2024-11-17T15:37:25,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for 34c72dd45fe3d9e6a1558a16ef53eb06: 2024-11-17T15:37:25,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. 2024-11-17T15:37:25,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-11-17T15:37:25,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-11-17T15:37:25,049 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-11-17T15:37:25,050 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3810 sec 2024-11-17T15:37:25,051 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees in 1.3860 sec 2024-11-17T15:37:25,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(8855): Flush requested on 34c72dd45fe3d9e6a1558a16ef53eb06 2024-11-17T15:37:25,425 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T15:37:25,425 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 34c72dd45fe3d9e6a1558a16ef53eb06 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-17T15:37:25,434 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411170860d2049ab943b4b091152722f08ad0_34c72dd45fe3d9e6a1558a16ef53eb06 is 50, key is test_row_0/A:col10/1731857844281/Put/seqid=0 2024-11-17T15:37:25,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742006_1182 (size=14794) 2024-11-17T15:37:25,452 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:25,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48408 deadline: 1731857905444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:25,453 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:25,453 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:25,453 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:25,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48438 deadline: 1731857905445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:25,453 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:25,453 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:25,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48420 deadline: 1731857905446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:25,453 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:25,453 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:25,453 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:25,453 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:25,453 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:25,453 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:25,458 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:25,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48428 deadline: 1731857905452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:25,458 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:25,458 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:25,458 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:25,495 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:25,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48396 deadline: 1731857905493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:25,496 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 7a780d55532c,32955,1731857800648 for 'test_row_0' in 34c72dd45fe3d9e6a1558a16ef53eb06 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4201 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T15:37:25,496 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:25,496 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:25,496 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:25,564 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:25,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48408 deadline: 1731857905563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:25,565 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:25,565 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:25,565 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:25,565 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:25,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48438 deadline: 1731857905563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:25,565 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:25,565 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:25,565 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:25,565 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:25,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48420 deadline: 1731857905563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:25,566 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:25,566 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:25,566 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:25,566 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:25,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48428 deadline: 1731857905563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:25,566 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:25,566 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:25,566 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:25,774 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:25,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48408 deadline: 1731857905773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:25,775 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:25,775 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:25,775 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:25,775 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:25,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48438 deadline: 1731857905773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:25,775 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:25,775 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:25,775 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:25,775 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:25,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48428 deadline: 1731857905773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:25,776 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:25,776 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:25,776 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:25,776 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:25,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48420 deadline: 1731857905773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:25,776 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:25,776 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:25,776 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:25,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-17T15:37:25,814 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T15:37:25,814 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T15:37:25,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees 2024-11-17T15:37:25,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-17T15:37:25,816 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T15:37:25,817 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T15:37:25,817 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T15:37:25,839 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:25,843 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411170860d2049ab943b4b091152722f08ad0_34c72dd45fe3d9e6a1558a16ef53eb06 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411170860d2049ab943b4b091152722f08ad0_34c72dd45fe3d9e6a1558a16ef53eb06 2024-11-17T15:37:25,845 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/A/dac104dc4ea04201915bef9a5cb3c8dc, store: [table=TestAcidGuarantees family=A region=34c72dd45fe3d9e6a1558a16ef53eb06] 2024-11-17T15:37:25,845 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/A/dac104dc4ea04201915bef9a5cb3c8dc is 175, key is test_row_0/A:col10/1731857844281/Put/seqid=0 2024-11-17T15:37:25,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742007_1183 (size=39749) 2024-11-17T15:37:25,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-17T15:37:25,970 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-17T15:37:25,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. 2024-11-17T15:37:25,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. as already flushing 2024-11-17T15:37:25,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. 2024-11-17T15:37:25,970 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 34c72dd45fe3d9e6a1558a16ef53eb06, NAME => 'TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:37:25,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 34c72dd45fe3d9e6a1558a16ef53eb06, NAME => 'TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:37:25,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 34c72dd45fe3d9e6a1558a16ef53eb06, NAME => 'TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 34c72dd45fe3d9e6a1558a16ef53eb06, NAME => 'TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:37:26,086 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:26,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48408 deadline: 1731857906083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:26,087 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:26,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48438 deadline: 1731857906083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:26,087 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:26,087 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:26,087 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:26,087 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:26,087 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:26,087 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:26,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48420 deadline: 1731857906083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:26,088 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:26,088 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:26,088 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:26,088 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:26,088 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:26,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48428 deadline: 1731857906083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:26,088 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:26,088 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:26,088 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:26,123 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-17T15:37:26,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. 2024-11-17T15:37:26,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. as already flushing 2024-11-17T15:37:26,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. 2024-11-17T15:37:26,124 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 34c72dd45fe3d9e6a1558a16ef53eb06, NAME => 'TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:37:26,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 34c72dd45fe3d9e6a1558a16ef53eb06, NAME => 'TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:37:26,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 34c72dd45fe3d9e6a1558a16ef53eb06, NAME => 'TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 34c72dd45fe3d9e6a1558a16ef53eb06, NAME => 'TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:37:26,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-17T15:37:26,250 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=256, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/A/dac104dc4ea04201915bef9a5cb3c8dc 2024-11-17T15:37:26,259 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/B/441d8e9769f7465aa080fa51197f4614 is 50, key is test_row_0/B:col10/1731857844281/Put/seqid=0 2024-11-17T15:37:26,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742008_1184 (size=12151) 2024-11-17T15:37:26,277 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-17T15:37:26,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. 2024-11-17T15:37:26,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. as already flushing 2024-11-17T15:37:26,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. 2024-11-17T15:37:26,277 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 34c72dd45fe3d9e6a1558a16ef53eb06, NAME => 'TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:37:26,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 34c72dd45fe3d9e6a1558a16ef53eb06, NAME => 'TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:37:26,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 34c72dd45fe3d9e6a1558a16ef53eb06, NAME => 'TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 34c72dd45fe3d9e6a1558a16ef53eb06, NAME => 'TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:37:26,430 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-17T15:37:26,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. 2024-11-17T15:37:26,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. as already flushing 2024-11-17T15:37:26,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. 2024-11-17T15:37:26,430 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 34c72dd45fe3d9e6a1558a16ef53eb06, NAME => 'TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:37:26,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 34c72dd45fe3d9e6a1558a16ef53eb06, NAME => 'TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:37:26,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 34c72dd45fe3d9e6a1558a16ef53eb06, NAME => 'TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 34c72dd45fe3d9e6a1558a16ef53eb06, NAME => 'TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:37:26,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-17T15:37:26,583 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-17T15:37:26,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. 2024-11-17T15:37:26,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. as already flushing 2024-11-17T15:37:26,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. 2024-11-17T15:37:26,584 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 34c72dd45fe3d9e6a1558a16ef53eb06, NAME => 'TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:37:26,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 34c72dd45fe3d9e6a1558a16ef53eb06, NAME => 'TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:37:26,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 34c72dd45fe3d9e6a1558a16ef53eb06, NAME => 'TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 34c72dd45fe3d9e6a1558a16ef53eb06, NAME => 'TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:37:26,596 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:26,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48408 deadline: 1731857906593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:26,596 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:26,596 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:26,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48420 deadline: 1731857906593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:26,597 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:26,597 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:26,597 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:26,597 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:26,597 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:26,605 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:26,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48438 deadline: 1731857906602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:26,606 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:26,606 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:26,606 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:26,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48428 deadline: 1731857906602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:26,606 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:26,606 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:26,606 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:26,606 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:26,663 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=256 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/B/441d8e9769f7465aa080fa51197f4614 2024-11-17T15:37:26,672 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/C/74e7ea14e44b49308a1750d48a1b8f2d is 50, key is test_row_0/C:col10/1731857844281/Put/seqid=0 2024-11-17T15:37:26,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742009_1185 (size=12151) 2024-11-17T15:37:26,679 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=256 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/C/74e7ea14e44b49308a1750d48a1b8f2d 2024-11-17T15:37:26,684 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/A/dac104dc4ea04201915bef9a5cb3c8dc as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/dac104dc4ea04201915bef9a5cb3c8dc 2024-11-17T15:37:26,688 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/dac104dc4ea04201915bef9a5cb3c8dc, entries=200, sequenceid=256, filesize=38.8 K 2024-11-17T15:37:26,689 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/B/441d8e9769f7465aa080fa51197f4614 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/B/441d8e9769f7465aa080fa51197f4614 2024-11-17T15:37:26,693 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/B/441d8e9769f7465aa080fa51197f4614, entries=150, sequenceid=256, filesize=11.9 K 2024-11-17T15:37:26,694 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/C/74e7ea14e44b49308a1750d48a1b8f2d as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/C/74e7ea14e44b49308a1750d48a1b8f2d 2024-11-17T15:37:26,699 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/C/74e7ea14e44b49308a1750d48a1b8f2d, entries=150, sequenceid=256, filesize=11.9 K 2024-11-17T15:37:26,700 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for 34c72dd45fe3d9e6a1558a16ef53eb06 in 1275ms, sequenceid=256, compaction requested=true 2024-11-17T15:37:26,700 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 34c72dd45fe3d9e6a1558a16ef53eb06: 2024-11-17T15:37:26,700 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 34c72dd45fe3d9e6a1558a16ef53eb06:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T15:37:26,700 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T15:37:26,700 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 34c72dd45fe3d9e6a1558a16ef53eb06:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T15:37:26,700 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T15:37:26,700 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T15:37:26,700 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T15:37:26,700 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 34c72dd45fe3d9e6a1558a16ef53eb06:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T15:37:26,700 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T15:37:26,702 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T15:37:26,702 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102471 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T15:37:26,702 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1541): 34c72dd45fe3d9e6a1558a16ef53eb06/A is initiating minor compaction (all files) 2024-11-17T15:37:26,702 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1541): 34c72dd45fe3d9e6a1558a16ef53eb06/B is initiating minor compaction (all files) 2024-11-17T15:37:26,702 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 34c72dd45fe3d9e6a1558a16ef53eb06/A in TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. 2024-11-17T15:37:26,702 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 34c72dd45fe3d9e6a1558a16ef53eb06/B in TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. 2024-11-17T15:37:26,702 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/819bfa332f2549c2af5a739a8fad5fdf, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/308900f4496248a285d0daabb50408cd, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/dac104dc4ea04201915bef9a5cb3c8dc] into tmpdir=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp, totalSize=100.1 K 2024-11-17T15:37:26,702 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/B/af2c4bc77ca34aae960590fc332a2a0f, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/B/6f8dbf9aa98d41fc9bf4c3a736f65eb4, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/B/441d8e9769f7465aa080fa51197f4614] into tmpdir=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp, totalSize=36.1 K 2024-11-17T15:37:26,702 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. 2024-11-17T15:37:26,702 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. files: [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/819bfa332f2549c2af5a739a8fad5fdf, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/308900f4496248a285d0daabb50408cd, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/dac104dc4ea04201915bef9a5cb3c8dc] 2024-11-17T15:37:26,703 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting af2c4bc77ca34aae960590fc332a2a0f, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1731857843596 2024-11-17T15:37:26,703 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 819bfa332f2549c2af5a739a8fad5fdf, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1731857843596 2024-11-17T15:37:26,703 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting 6f8dbf9aa98d41fc9bf4c3a736f65eb4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1731857843600 2024-11-17T15:37:26,703 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 308900f4496248a285d0daabb50408cd, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1731857843600 2024-11-17T15:37:26,704 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting 441d8e9769f7465aa080fa51197f4614, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1731857844281 2024-11-17T15:37:26,704 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting dac104dc4ea04201915bef9a5cb3c8dc, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1731857844281 2024-11-17T15:37:26,713 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=34c72dd45fe3d9e6a1558a16ef53eb06] 2024-11-17T15:37:26,716 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 34c72dd45fe3d9e6a1558a16ef53eb06#B#compaction#153 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T15:37:26,717 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411176b151930067346f6bfef2c372f51d128_34c72dd45fe3d9e6a1558a16ef53eb06 store=[table=TestAcidGuarantees family=A region=34c72dd45fe3d9e6a1558a16ef53eb06] 2024-11-17T15:37:26,717 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/B/e01856a38d7440b6ada86832558eb10a is 50, key is test_row_0/B:col10/1731857844281/Put/seqid=0 2024-11-17T15:37:26,719 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411176b151930067346f6bfef2c372f51d128_34c72dd45fe3d9e6a1558a16ef53eb06, store=[table=TestAcidGuarantees family=A region=34c72dd45fe3d9e6a1558a16ef53eb06] 2024-11-17T15:37:26,719 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411176b151930067346f6bfef2c372f51d128_34c72dd45fe3d9e6a1558a16ef53eb06 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=34c72dd45fe3d9e6a1558a16ef53eb06] 2024-11-17T15:37:26,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742010_1186 (size=12765) 2024-11-17T15:37:26,736 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-17T15:37:26,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. 2024-11-17T15:37:26,737 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2902): Flushing 34c72dd45fe3d9e6a1558a16ef53eb06 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-17T15:37:26,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742011_1187 (size=4469) 2024-11-17T15:37:26,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411173f22668ee10346949d5fda6524672a93_34c72dd45fe3d9e6a1558a16ef53eb06 is 50, key is test_row_0/A:col10/1731857845451/Put/seqid=0 2024-11-17T15:37:26,763 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742012_1188 (size=12454) 2024-11-17T15:37:26,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-17T15:37:27,142 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/B/e01856a38d7440b6ada86832558eb10a as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/B/e01856a38d7440b6ada86832558eb10a 2024-11-17T15:37:27,142 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 34c72dd45fe3d9e6a1558a16ef53eb06#A#compaction#152 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T15:37:27,143 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/A/1f2b7a1bc19a4e4d97d609e51b76d0ab is 175, key is test_row_0/A:col10/1731857844281/Put/seqid=0 2024-11-17T15:37:27,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742013_1189 (size=31719) 2024-11-17T15:37:27,151 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 34c72dd45fe3d9e6a1558a16ef53eb06/B of 34c72dd45fe3d9e6a1558a16ef53eb06 into e01856a38d7440b6ada86832558eb10a(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T15:37:27,151 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 34c72dd45fe3d9e6a1558a16ef53eb06: 2024-11-17T15:37:27,151 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., storeName=34c72dd45fe3d9e6a1558a16ef53eb06/B, priority=13, startTime=1731857846700; duration=0sec 2024-11-17T15:37:27,151 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T15:37:27,151 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 34c72dd45fe3d9e6a1558a16ef53eb06:B 2024-11-17T15:37:27,151 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T15:37:27,153 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T15:37:27,153 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1541): 34c72dd45fe3d9e6a1558a16ef53eb06/C is initiating minor compaction (all files) 2024-11-17T15:37:27,153 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 34c72dd45fe3d9e6a1558a16ef53eb06/C in TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. 2024-11-17T15:37:27,153 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/C/99c73440b67a416d9cf345e797bcda0a, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/C/72d4f83ff53e4106ac571c9507fa64fb, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/C/74e7ea14e44b49308a1750d48a1b8f2d] into tmpdir=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp, totalSize=36.1 K 2024-11-17T15:37:27,153 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/A/1f2b7a1bc19a4e4d97d609e51b76d0ab as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/1f2b7a1bc19a4e4d97d609e51b76d0ab 2024-11-17T15:37:27,154 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting 99c73440b67a416d9cf345e797bcda0a, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1731857843596 2024-11-17T15:37:27,154 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting 72d4f83ff53e4106ac571c9507fa64fb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1731857843600 2024-11-17T15:37:27,155 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting 74e7ea14e44b49308a1750d48a1b8f2d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1731857844281 2024-11-17T15:37:27,159 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 34c72dd45fe3d9e6a1558a16ef53eb06/A of 34c72dd45fe3d9e6a1558a16ef53eb06 into 1f2b7a1bc19a4e4d97d609e51b76d0ab(size=31.0 K), total size for store is 31.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T15:37:27,159 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 34c72dd45fe3d9e6a1558a16ef53eb06: 2024-11-17T15:37:27,159 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., storeName=34c72dd45fe3d9e6a1558a16ef53eb06/A, priority=13, startTime=1731857846700; duration=0sec 2024-11-17T15:37:27,159 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T15:37:27,159 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 34c72dd45fe3d9e6a1558a16ef53eb06:A 2024-11-17T15:37:27,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:27,166 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 34c72dd45fe3d9e6a1558a16ef53eb06#C#compaction#155 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T15:37:27,167 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/C/3201087044614829893329419faa4487 is 50, key is test_row_0/C:col10/1731857844281/Put/seqid=0 2024-11-17T15:37:27,167 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411173f22668ee10346949d5fda6524672a93_34c72dd45fe3d9e6a1558a16ef53eb06 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411173f22668ee10346949d5fda6524672a93_34c72dd45fe3d9e6a1558a16ef53eb06 2024-11-17T15:37:27,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/A/8d545cfa415f4bb1b137813b2a6c0ac8, store: [table=TestAcidGuarantees family=A region=34c72dd45fe3d9e6a1558a16ef53eb06] 2024-11-17T15:37:27,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/A/8d545cfa415f4bb1b137813b2a6c0ac8 is 175, key is test_row_0/A:col10/1731857845451/Put/seqid=0 2024-11-17T15:37:27,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742015_1191 (size=31255) 2024-11-17T15:37:27,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742014_1190 (size=12731) 2024-11-17T15:37:27,176 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=271, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/A/8d545cfa415f4bb1b137813b2a6c0ac8 2024-11-17T15:37:27,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/B/5319f67f29c14f43925cda63d7a7e216 is 50, key is test_row_0/B:col10/1731857845451/Put/seqid=0 2024-11-17T15:37:27,187 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742016_1192 (size=12301) 2024-11-17T15:37:27,188 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=271 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/B/5319f67f29c14f43925cda63d7a7e216 2024-11-17T15:37:27,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/C/dc79b2f01ff542fca414e2d9754bba08 is 50, key is test_row_0/C:col10/1731857845451/Put/seqid=0 2024-11-17T15:37:27,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742017_1193 (size=12301) 2024-11-17T15:37:27,582 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/C/3201087044614829893329419faa4487 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/C/3201087044614829893329419faa4487 2024-11-17T15:37:27,587 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 34c72dd45fe3d9e6a1558a16ef53eb06/C of 34c72dd45fe3d9e6a1558a16ef53eb06 into 3201087044614829893329419faa4487(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T15:37:27,587 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 34c72dd45fe3d9e6a1558a16ef53eb06: 2024-11-17T15:37:27,587 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., storeName=34c72dd45fe3d9e6a1558a16ef53eb06/C, priority=13, startTime=1731857846700; duration=0sec 2024-11-17T15:37:27,587 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T15:37:27,587 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 34c72dd45fe3d9e6a1558a16ef53eb06:C 2024-11-17T15:37:27,601 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=271 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/C/dc79b2f01ff542fca414e2d9754bba08 2024-11-17T15:37:27,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(8855): Flush requested on 34c72dd45fe3d9e6a1558a16ef53eb06 2024-11-17T15:37:27,605 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. as already flushing 2024-11-17T15:37:27,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/A/8d545cfa415f4bb1b137813b2a6c0ac8 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/8d545cfa415f4bb1b137813b2a6c0ac8 2024-11-17T15:37:27,611 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/8d545cfa415f4bb1b137813b2a6c0ac8, entries=150, sequenceid=271, filesize=30.5 K 2024-11-17T15:37:27,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/B/5319f67f29c14f43925cda63d7a7e216 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/B/5319f67f29c14f43925cda63d7a7e216 2024-11-17T15:37:27,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:27,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:27,619 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/B/5319f67f29c14f43925cda63d7a7e216, entries=150, sequenceid=271, filesize=12.0 K 2024-11-17T15:37:27,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:27,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/C/dc79b2f01ff542fca414e2d9754bba08 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/C/dc79b2f01ff542fca414e2d9754bba08 2024-11-17T15:37:27,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:27,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:27,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:27,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:27,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:27,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:27,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:27,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:27,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:27,627 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/C/dc79b2f01ff542fca414e2d9754bba08, entries=150, sequenceid=271, filesize=12.0 K 2024-11-17T15:37:27,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:27,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:27,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:27,628 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=46.96 KB/48090 for 34c72dd45fe3d9e6a1558a16ef53eb06 in 891ms, sequenceid=271, compaction requested=false 2024-11-17T15:37:27,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2603): Flush status journal for 34c72dd45fe3d9e6a1558a16ef53eb06: 2024-11-17T15:37:27,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. 2024-11-17T15:37:27,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=53 2024-11-17T15:37:27,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:27,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.HMaster(4169): Remote procedure done, pid=53 2024-11-17T15:37:27,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:27,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:27,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:27,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:27,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:27,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:27,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:27,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:27,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:27,632 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-11-17T15:37:27,632 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8130 sec 2024-11-17T15:37:27,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:27,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:27,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:27,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:27,634 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees in 1.8190 sec 2024-11-17T15:37:27,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:27,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:27,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:27,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:27,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:27,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:27,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:27,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:27,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:27,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:27,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:27,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:27,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:27,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(8855): Flush requested on 34c72dd45fe3d9e6a1558a16ef53eb06 2024-11-17T15:37:27,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:27,640 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T15:37:27,640 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 34c72dd45fe3d9e6a1558a16ef53eb06 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-17T15:37:27,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:27,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:27,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:27,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:27,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:27,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:27,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:27,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:27,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:27,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:27,651 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111784b26e095b0c4ece98c460c0b41eba67_34c72dd45fe3d9e6a1558a16ef53eb06 is 50, key is test_row_0/A:col10/1731857847639/Put/seqid=0 2024-11-17T15:37:27,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:27,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:27,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:27,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:27,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:27,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:27,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:27,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:27,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:27,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:27,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:27,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:27,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:27,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:27,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:27,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:27,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:27,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:27,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:27,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:27,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:27,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:27,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:27,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:27,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:27,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:27,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:27,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:27,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:27,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:27,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:27,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:27,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:27,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:27,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:27,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:27,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:27,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:27,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:27,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:27,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:27,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:27,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:27,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:27,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:27,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:27,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:27,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742018_1194 (size=22618) 2024-11-17T15:37:27,678 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:27,683 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111784b26e095b0c4ece98c460c0b41eba67_34c72dd45fe3d9e6a1558a16ef53eb06 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111784b26e095b0c4ece98c460c0b41eba67_34c72dd45fe3d9e6a1558a16ef53eb06 2024-11-17T15:37:27,684 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/A/47eb3c5928e94409b3b2897c3eeedcaa, store: [table=TestAcidGuarantees family=A region=34c72dd45fe3d9e6a1558a16ef53eb06] 2024-11-17T15:37:27,685 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/A/47eb3c5928e94409b3b2897c3eeedcaa is 175, key is test_row_0/A:col10/1731857847639/Put/seqid=0 2024-11-17T15:37:27,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742019_1195 (size=66023) 2024-11-17T15:37:27,701 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=285, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/A/47eb3c5928e94409b3b2897c3eeedcaa 2024-11-17T15:37:27,714 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/B/f6619f057a24400ba950207261102d13 is 50, key is test_row_0/B:col10/1731857847639/Put/seqid=0 2024-11-17T15:37:27,731 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742020_1196 (size=12301) 2024-11-17T15:37:27,732 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=285 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/B/f6619f057a24400ba950207261102d13 2024-11-17T15:37:27,738 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:27,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48420 deadline: 1731857907724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:27,738 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:27,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48408 deadline: 1731857907726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:27,738 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:27,738 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:27,738 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:27,739 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:27,739 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:27,739 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:27,738 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:27,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48438 deadline: 1731857907721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:27,742 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/C/4cbd51af63d84631a34bc1c8ff659f16 is 50, key is test_row_0/C:col10/1731857847639/Put/seqid=0 2024-11-17T15:37:27,743 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:27,743 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:27,743 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:27,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742021_1197 (size=12301) 2024-11-17T15:37:27,751 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:27,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48428 deadline: 1731857907737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:27,752 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=285 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/C/4cbd51af63d84631a34bc1c8ff659f16 2024-11-17T15:37:27,752 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:27,752 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:27,752 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:27,761 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/A/47eb3c5928e94409b3b2897c3eeedcaa as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/47eb3c5928e94409b3b2897c3eeedcaa 2024-11-17T15:37:27,771 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/47eb3c5928e94409b3b2897c3eeedcaa, entries=350, sequenceid=285, filesize=64.5 K 2024-11-17T15:37:27,771 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/B/f6619f057a24400ba950207261102d13 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/B/f6619f057a24400ba950207261102d13 2024-11-17T15:37:27,776 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/B/f6619f057a24400ba950207261102d13, entries=150, sequenceid=285, filesize=12.0 K 2024-11-17T15:37:27,777 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/C/4cbd51af63d84631a34bc1c8ff659f16 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/C/4cbd51af63d84631a34bc1c8ff659f16 2024-11-17T15:37:27,782 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/C/4cbd51af63d84631a34bc1c8ff659f16, entries=150, sequenceid=285, filesize=12.0 K 2024-11-17T15:37:27,783 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 34c72dd45fe3d9e6a1558a16ef53eb06 in 143ms, sequenceid=285, compaction requested=true 2024-11-17T15:37:27,783 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 34c72dd45fe3d9e6a1558a16ef53eb06: 2024-11-17T15:37:27,783 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 34c72dd45fe3d9e6a1558a16ef53eb06:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T15:37:27,783 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T15:37:27,783 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T15:37:27,783 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 34c72dd45fe3d9e6a1558a16ef53eb06:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T15:37:27,783 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T15:37:27,783 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 34c72dd45fe3d9e6a1558a16ef53eb06:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T15:37:27,783 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-17T15:37:27,783 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T15:37:27,785 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 128997 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T15:37:27,785 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37367 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T15:37:27,785 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1541): 34c72dd45fe3d9e6a1558a16ef53eb06/A is initiating minor compaction (all files) 2024-11-17T15:37:27,785 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1541): 34c72dd45fe3d9e6a1558a16ef53eb06/B is initiating minor compaction (all files) 2024-11-17T15:37:27,785 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 34c72dd45fe3d9e6a1558a16ef53eb06/B in TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. 2024-11-17T15:37:27,785 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 34c72dd45fe3d9e6a1558a16ef53eb06/A in TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. 2024-11-17T15:37:27,785 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/B/e01856a38d7440b6ada86832558eb10a, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/B/5319f67f29c14f43925cda63d7a7e216, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/B/f6619f057a24400ba950207261102d13] into tmpdir=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp, totalSize=36.5 K 2024-11-17T15:37:27,785 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/1f2b7a1bc19a4e4d97d609e51b76d0ab, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/8d545cfa415f4bb1b137813b2a6c0ac8, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/47eb3c5928e94409b3b2897c3eeedcaa] into tmpdir=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp, totalSize=126.0 K 2024-11-17T15:37:27,785 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. 2024-11-17T15:37:27,785 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. files: [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/1f2b7a1bc19a4e4d97d609e51b76d0ab, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/8d545cfa415f4bb1b137813b2a6c0ac8, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/47eb3c5928e94409b3b2897c3eeedcaa] 2024-11-17T15:37:27,785 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting e01856a38d7440b6ada86832558eb10a, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1731857844281 2024-11-17T15:37:27,785 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1f2b7a1bc19a4e4d97d609e51b76d0ab, keycount=150, bloomtype=ROW, size=31.0 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1731857844281 2024-11-17T15:37:27,786 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting 5319f67f29c14f43925cda63d7a7e216, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1731857845435 2024-11-17T15:37:27,786 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8d545cfa415f4bb1b137813b2a6c0ac8, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1731857845435 2024-11-17T15:37:27,786 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting f6619f057a24400ba950207261102d13, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1731857847620 2024-11-17T15:37:27,786 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 47eb3c5928e94409b3b2897c3eeedcaa, keycount=350, bloomtype=ROW, size=64.5 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1731857847606 2024-11-17T15:37:27,798 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=34c72dd45fe3d9e6a1558a16ef53eb06] 2024-11-17T15:37:27,800 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 34c72dd45fe3d9e6a1558a16ef53eb06#B#compaction#161 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T15:37:27,800 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/B/f1ae25541f53409dbb5c19fb9d4e82bd is 50, key is test_row_0/B:col10/1731857847639/Put/seqid=0 2024-11-17T15:37:27,818 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411174d850889cf224ab093e2056f5ebc6bb1_34c72dd45fe3d9e6a1558a16ef53eb06 store=[table=TestAcidGuarantees family=A region=34c72dd45fe3d9e6a1558a16ef53eb06] 2024-11-17T15:37:27,824 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411174d850889cf224ab093e2056f5ebc6bb1_34c72dd45fe3d9e6a1558a16ef53eb06, store=[table=TestAcidGuarantees family=A region=34c72dd45fe3d9e6a1558a16ef53eb06] 2024-11-17T15:37:27,825 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411174d850889cf224ab093e2056f5ebc6bb1_34c72dd45fe3d9e6a1558a16ef53eb06 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=34c72dd45fe3d9e6a1558a16ef53eb06] 2024-11-17T15:37:27,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742022_1198 (size=13017) 2024-11-17T15:37:27,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742023_1199 (size=4469) 2024-11-17T15:37:27,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(8855): Flush requested on 34c72dd45fe3d9e6a1558a16ef53eb06 2024-11-17T15:37:27,845 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T15:37:27,845 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 34c72dd45fe3d9e6a1558a16ef53eb06 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-17T15:37:27,864 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:27,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48438 deadline: 1731857907858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:27,864 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:27,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48420 deadline: 1731857907859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:27,865 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:27,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48408 deadline: 1731857907862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:27,865 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:27,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48428 deadline: 1731857907863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:27,865 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:27,865 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:27,865 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:27,867 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411177c90f771257d495f8625cad85d6ded06_34c72dd45fe3d9e6a1558a16ef53eb06 is 50, key is test_row_0/A:col10/1731857847844/Put/seqid=0 2024-11-17T15:37:27,867 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:27,867 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:27,867 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:27,868 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:27,868 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:27,868 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:27,868 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:27,868 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:27,868 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:27,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742024_1200 (size=12454) 2024-11-17T15:37:27,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-17T15:37:27,963 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T15:37:27,964 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T15:37:27,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees 2024-11-17T15:37:27,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-17T15:37:27,966 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T15:37:27,967 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T15:37:27,967 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T15:37:27,974 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:27,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48438 deadline: 1731857907973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:27,974 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:27,974 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:27,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48408 deadline: 1731857907973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:27,974 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:27,974 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:27,975 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:27,975 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:27,975 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:27,975 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:27,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48420 deadline: 1731857907973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:27,975 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:27,975 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:27,975 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:28,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-17T15:37:28,075 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:28,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48428 deadline: 1731857908073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:28,075 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:28,075 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:28,075 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:28,120 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-17T15:37:28,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. 2024-11-17T15:37:28,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. as already flushing 2024-11-17T15:37:28,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. 2024-11-17T15:37:28,120 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 34c72dd45fe3d9e6a1558a16ef53eb06, NAME => 'TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:37:28,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 34c72dd45fe3d9e6a1558a16ef53eb06, NAME => 'TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:37:28,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 34c72dd45fe3d9e6a1558a16ef53eb06, NAME => 'TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 34c72dd45fe3d9e6a1558a16ef53eb06, NAME => 'TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:37:28,191 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:28,192 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:28,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48438 deadline: 1731857908183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:28,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48408 deadline: 1731857908183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:28,192 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:28,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48420 deadline: 1731857908183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:28,192 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:28,192 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:28,192 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:28,192 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:28,192 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:28,192 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:28,192 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:28,192 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:28,192 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:28,244 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 34c72dd45fe3d9e6a1558a16ef53eb06#A#compaction#162 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T15:37:28,244 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/B/f1ae25541f53409dbb5c19fb9d4e82bd as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/B/f1ae25541f53409dbb5c19fb9d4e82bd 2024-11-17T15:37:28,244 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/A/da3ea977d6e64552982debb9e49a0656 is 175, key is test_row_0/A:col10/1731857847639/Put/seqid=0 2024-11-17T15:37:28,249 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742025_1201 (size=31971) 2024-11-17T15:37:28,250 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 34c72dd45fe3d9e6a1558a16ef53eb06/B of 34c72dd45fe3d9e6a1558a16ef53eb06 into f1ae25541f53409dbb5c19fb9d4e82bd(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T15:37:28,250 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 34c72dd45fe3d9e6a1558a16ef53eb06: 2024-11-17T15:37:28,251 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., storeName=34c72dd45fe3d9e6a1558a16ef53eb06/B, priority=13, startTime=1731857847783; duration=0sec 2024-11-17T15:37:28,251 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T15:37:28,251 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 34c72dd45fe3d9e6a1558a16ef53eb06:B 2024-11-17T15:37:28,251 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T15:37:28,252 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T15:37:28,252 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1541): 34c72dd45fe3d9e6a1558a16ef53eb06/C is initiating minor compaction (all files) 2024-11-17T15:37:28,252 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 34c72dd45fe3d9e6a1558a16ef53eb06/C in TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. 2024-11-17T15:37:28,252 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/C/3201087044614829893329419faa4487, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/C/dc79b2f01ff542fca414e2d9754bba08, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/C/4cbd51af63d84631a34bc1c8ff659f16] into tmpdir=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp, totalSize=36.5 K 2024-11-17T15:37:28,252 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting 3201087044614829893329419faa4487, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1731857844281 2024-11-17T15:37:28,253 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting dc79b2f01ff542fca414e2d9754bba08, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1731857845435 2024-11-17T15:37:28,253 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting 4cbd51af63d84631a34bc1c8ff659f16, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1731857847620 2024-11-17T15:37:28,261 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 34c72dd45fe3d9e6a1558a16ef53eb06#C#compaction#164 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T15:37:28,261 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/C/2e128634176d46d9af45f70a8e60e2cc is 50, key is test_row_0/C:col10/1731857847639/Put/seqid=0 2024-11-17T15:37:28,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742026_1202 (size=12983) 2024-11-17T15:37:28,273 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-17T15:37:28,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. 2024-11-17T15:37:28,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. as already flushing 2024-11-17T15:37:28,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. 2024-11-17T15:37:28,274 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 34c72dd45fe3d9e6a1558a16ef53eb06, NAME => 'TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:37:28,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 34c72dd45fe3d9e6a1558a16ef53eb06, NAME => 'TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:37:28,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 34c72dd45fe3d9e6a1558a16ef53eb06, NAME => 'TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 34c72dd45fe3d9e6a1558a16ef53eb06, NAME => 'TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:37:28,277 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/C/2e128634176d46d9af45f70a8e60e2cc as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/C/2e128634176d46d9af45f70a8e60e2cc 2024-11-17T15:37:28,282 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 34c72dd45fe3d9e6a1558a16ef53eb06/C of 34c72dd45fe3d9e6a1558a16ef53eb06 into 2e128634176d46d9af45f70a8e60e2cc(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T15:37:28,282 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 34c72dd45fe3d9e6a1558a16ef53eb06: 2024-11-17T15:37:28,282 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., storeName=34c72dd45fe3d9e6a1558a16ef53eb06/C, priority=13, startTime=1731857847783; duration=0sec 2024-11-17T15:37:28,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-17T15:37:28,282 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T15:37:28,283 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 34c72dd45fe3d9e6a1558a16ef53eb06:C 2024-11-17T15:37:28,283 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:28,287 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411177c90f771257d495f8625cad85d6ded06_34c72dd45fe3d9e6a1558a16ef53eb06 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411177c90f771257d495f8625cad85d6ded06_34c72dd45fe3d9e6a1558a16ef53eb06 2024-11-17T15:37:28,288 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/A/93ca7622c0534237856b58aee1fddeba, store: [table=TestAcidGuarantees family=A region=34c72dd45fe3d9e6a1558a16ef53eb06] 2024-11-17T15:37:28,289 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/A/93ca7622c0534237856b58aee1fddeba is 175, key is test_row_0/A:col10/1731857847844/Put/seqid=0 2024-11-17T15:37:28,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742027_1203 (size=31255) 2024-11-17T15:37:28,387 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:28,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48428 deadline: 1731857908383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:28,387 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:28,388 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:28,388 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:28,427 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-17T15:37:28,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. 2024-11-17T15:37:28,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. as already flushing 2024-11-17T15:37:28,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. 2024-11-17T15:37:28,427 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 34c72dd45fe3d9e6a1558a16ef53eb06, NAME => 'TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:37:28,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 34c72dd45fe3d9e6a1558a16ef53eb06, NAME => 'TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:37:28,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 34c72dd45fe3d9e6a1558a16ef53eb06, NAME => 'TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 34c72dd45fe3d9e6a1558a16ef53eb06, NAME => 'TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:37:28,503 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:28,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48438 deadline: 1731857908503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:28,504 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:28,504 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:28,504 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:28,504 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:28,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48408 deadline: 1731857908503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:28,504 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:28,504 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:28,504 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:28,509 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:28,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48420 deadline: 1731857908503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:28,509 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:28,509 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:28,510 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:28,580 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-17T15:37:28,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. 2024-11-17T15:37:28,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. as already flushing 2024-11-17T15:37:28,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. 2024-11-17T15:37:28,581 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 34c72dd45fe3d9e6a1558a16ef53eb06, NAME => 'TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:37:28,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 34c72dd45fe3d9e6a1558a16ef53eb06, NAME => 'TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:37:28,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 34c72dd45fe3d9e6a1558a16ef53eb06, NAME => 'TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 34c72dd45fe3d9e6a1558a16ef53eb06, NAME => 'TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:37:28,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-17T15:37:28,656 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/A/da3ea977d6e64552982debb9e49a0656 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/da3ea977d6e64552982debb9e49a0656 2024-11-17T15:37:28,661 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 34c72dd45fe3d9e6a1558a16ef53eb06/A of 34c72dd45fe3d9e6a1558a16ef53eb06 into da3ea977d6e64552982debb9e49a0656(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T15:37:28,661 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 34c72dd45fe3d9e6a1558a16ef53eb06: 2024-11-17T15:37:28,661 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., storeName=34c72dd45fe3d9e6a1558a16ef53eb06/A, priority=13, startTime=1731857847783; duration=0sec 2024-11-17T15:37:28,661 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T15:37:28,661 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 34c72dd45fe3d9e6a1558a16ef53eb06:A 2024-11-17T15:37:28,693 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=312, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/A/93ca7622c0534237856b58aee1fddeba 2024-11-17T15:37:28,701 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/B/7cd42ed066324fc4bcf1d037ea6871da is 50, key is test_row_0/B:col10/1731857847844/Put/seqid=0 2024-11-17T15:37:28,705 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742028_1204 (size=12301) 2024-11-17T15:37:28,706 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=312 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/B/7cd42ed066324fc4bcf1d037ea6871da 2024-11-17T15:37:28,714 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/C/595f978bb41141eeb3c944ed33e4fe09 is 50, key is test_row_0/C:col10/1731857847844/Put/seqid=0 2024-11-17T15:37:28,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742029_1205 (size=12301) 2024-11-17T15:37:28,734 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-17T15:37:28,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. 2024-11-17T15:37:28,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. as already flushing 2024-11-17T15:37:28,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. 2024-11-17T15:37:28,735 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 34c72dd45fe3d9e6a1558a16ef53eb06, NAME => 'TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:37:28,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 34c72dd45fe3d9e6a1558a16ef53eb06, NAME => 'TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:37:28,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 34c72dd45fe3d9e6a1558a16ef53eb06, NAME => 'TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 34c72dd45fe3d9e6a1558a16ef53eb06, NAME => 'TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:37:28,888 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-17T15:37:28,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. 2024-11-17T15:37:28,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. as already flushing 2024-11-17T15:37:28,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. 2024-11-17T15:37:28,888 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 34c72dd45fe3d9e6a1558a16ef53eb06, NAME => 'TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:37:28,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 34c72dd45fe3d9e6a1558a16ef53eb06, NAME => 'TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:37:28,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 34c72dd45fe3d9e6a1558a16ef53eb06, NAME => 'TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 34c72dd45fe3d9e6a1558a16ef53eb06, NAME => 'TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:37:28,897 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:28,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48428 deadline: 1731857908893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:28,897 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:28,898 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:28,898 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:29,019 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:29,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48438 deadline: 1731857909013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:29,019 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:29,019 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:29,019 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:29,019 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:29,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48408 deadline: 1731857909013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:29,020 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:29,020 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:29,020 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:29,020 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:29,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48420 deadline: 1731857909013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:29,020 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:29,020 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:29,020 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:29,041 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-17T15:37:29,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. 2024-11-17T15:37:29,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. as already flushing 2024-11-17T15:37:29,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. 2024-11-17T15:37:29,041 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 34c72dd45fe3d9e6a1558a16ef53eb06, NAME => 'TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:37:29,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 34c72dd45fe3d9e6a1558a16ef53eb06, NAME => 'TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:37:29,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 34c72dd45fe3d9e6a1558a16ef53eb06, NAME => 'TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 34c72dd45fe3d9e6a1558a16ef53eb06, NAME => 'TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:37:29,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-17T15:37:29,119 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=312 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/C/595f978bb41141eeb3c944ed33e4fe09 2024-11-17T15:37:29,125 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/A/93ca7622c0534237856b58aee1fddeba as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/93ca7622c0534237856b58aee1fddeba 2024-11-17T15:37:29,129 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/93ca7622c0534237856b58aee1fddeba, entries=150, sequenceid=312, filesize=30.5 K 2024-11-17T15:37:29,130 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/B/7cd42ed066324fc4bcf1d037ea6871da as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/B/7cd42ed066324fc4bcf1d037ea6871da 2024-11-17T15:37:29,134 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/B/7cd42ed066324fc4bcf1d037ea6871da, entries=150, sequenceid=312, filesize=12.0 K 2024-11-17T15:37:29,135 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/C/595f978bb41141eeb3c944ed33e4fe09 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/C/595f978bb41141eeb3c944ed33e4fe09 2024-11-17T15:37:29,139 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/C/595f978bb41141eeb3c944ed33e4fe09, entries=150, sequenceid=312, filesize=12.0 K 2024-11-17T15:37:29,140 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for 34c72dd45fe3d9e6a1558a16ef53eb06 in 1295ms, sequenceid=312, compaction requested=false 2024-11-17T15:37:29,140 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 34c72dd45fe3d9e6a1558a16ef53eb06: 2024-11-17T15:37:29,194 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-17T15:37:29,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. 2024-11-17T15:37:29,194 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2902): Flushing 34c72dd45fe3d9e6a1558a16ef53eb06 3/3 column families, dataSize=40.25 KB heapSize=106.22 KB 2024-11-17T15:37:29,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111700b015045823475784c0167ef6f9d141_34c72dd45fe3d9e6a1558a16ef53eb06 is 50, key is test_row_0/A:col10/1731857847860/Put/seqid=0 2024-11-17T15:37:29,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742030_1206 (size=12454) 2024-11-17T15:37:29,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:29,210 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111700b015045823475784c0167ef6f9d141_34c72dd45fe3d9e6a1558a16ef53eb06 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111700b015045823475784c0167ef6f9d141_34c72dd45fe3d9e6a1558a16ef53eb06 2024-11-17T15:37:29,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/A/e4c877389119447d9e92b422daeed890, store: [table=TestAcidGuarantees family=A region=34c72dd45fe3d9e6a1558a16ef53eb06] 2024-11-17T15:37:29,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/A/e4c877389119447d9e92b422daeed890 is 175, key is test_row_0/A:col10/1731857847860/Put/seqid=0 2024-11-17T15:37:29,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742031_1207 (size=31255) 2024-11-17T15:37:29,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(8855): Flush requested on 34c72dd45fe3d9e6a1558a16ef53eb06 2024-11-17T15:37:29,528 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. as already flushing 2024-11-17T15:37:29,606 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:29,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48396 deadline: 1731857909603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:29,606 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:29,606 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:29,607 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:29,627 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=324, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/A/e4c877389119447d9e92b422daeed890 2024-11-17T15:37:29,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/B/f3a4d63dcd98420b8bb9808dd58eb23f is 50, key is test_row_0/B:col10/1731857847860/Put/seqid=0 2024-11-17T15:37:29,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742032_1208 (size=12301) 2024-11-17T15:37:29,714 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:29,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48396 deadline: 1731857909713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:29,715 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:29,715 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:29,715 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:29,914 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:29,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48428 deadline: 1731857909913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:29,915 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 7a780d55532c,32955,1731857800648 for 'test_row_2' in 34c72dd45fe3d9e6a1558a16ef53eb06 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2178 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T15:37:29,916 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:29,916 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:29,916 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:29,923 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:29,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48396 deadline: 1731857909922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:29,924 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:29,924 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:29,924 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:30,027 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:30,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48438 deadline: 1731857910024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:30,027 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:30,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48408 deadline: 1731857910024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:30,028 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 7a780d55532c,32955,1731857800648 for 'test_row_1' in 34c72dd45fe3d9e6a1558a16ef53eb06 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T15:37:30,028 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:30,028 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:30,028 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:30,028 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 7a780d55532c,32955,1731857800648 for 'test_row_1' in 34c72dd45fe3d9e6a1558a16ef53eb06 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2166 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T15:37:30,029 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:30,029 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:30,029 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:30,033 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:30,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48420 deadline: 1731857910033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:30,034 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 7a780d55532c,32955,1731857800648 for 'test_row_1' in 34c72dd45fe3d9e6a1558a16ef53eb06 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2174 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T15:37:30,034 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:30,034 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:30,034 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:30,040 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=324 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/B/f3a4d63dcd98420b8bb9808dd58eb23f 2024-11-17T15:37:30,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/C/9de6f2908dbb49c684ca539e76317197 is 50, key is test_row_0/C:col10/1731857847860/Put/seqid=0 2024-11-17T15:37:30,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742033_1209 (size=12301) 2024-11-17T15:37:30,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-17T15:37:30,237 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:30,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48396 deadline: 1731857910233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:30,237 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:30,237 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:30,237 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:30,455 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=324 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/C/9de6f2908dbb49c684ca539e76317197 2024-11-17T15:37:30,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/A/e4c877389119447d9e92b422daeed890 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/e4c877389119447d9e92b422daeed890 2024-11-17T15:37:30,464 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/e4c877389119447d9e92b422daeed890, entries=150, sequenceid=324, filesize=30.5 K 2024-11-17T15:37:30,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/B/f3a4d63dcd98420b8bb9808dd58eb23f as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/B/f3a4d63dcd98420b8bb9808dd58eb23f 2024-11-17T15:37:30,469 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/B/f3a4d63dcd98420b8bb9808dd58eb23f, entries=150, sequenceid=324, filesize=12.0 K 2024-11-17T15:37:30,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/C/9de6f2908dbb49c684ca539e76317197 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/C/9de6f2908dbb49c684ca539e76317197 2024-11-17T15:37:30,474 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/C/9de6f2908dbb49c684ca539e76317197, entries=150, sequenceid=324, filesize=12.0 K 2024-11-17T15:37:30,475 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.17 KB/108720, currentSize=161.02 KB/164880 for 34c72dd45fe3d9e6a1558a16ef53eb06 in 1281ms, sequenceid=324, compaction requested=true 2024-11-17T15:37:30,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2603): Flush status journal for 34c72dd45fe3d9e6a1558a16ef53eb06: 2024-11-17T15:37:30,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. 2024-11-17T15:37:30,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=55 2024-11-17T15:37:30,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.HMaster(4169): Remote procedure done, pid=55 2024-11-17T15:37:30,478 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-11-17T15:37:30,478 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5090 sec 2024-11-17T15:37:30,479 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees in 2.5140 sec 2024-11-17T15:37:30,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(8855): Flush requested on 34c72dd45fe3d9e6a1558a16ef53eb06 2024-11-17T15:37:30,746 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T15:37:30,746 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 34c72dd45fe3d9e6a1558a16ef53eb06 3/3 column families, dataSize=167.72 KB heapSize=440.20 KB 2024-11-17T15:37:30,754 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411179c5f48f2ffff4c5c8622c4482c9f36a0_34c72dd45fe3d9e6a1558a16ef53eb06 is 50, key is test_row_0/A:col10/1731857850744/Put/seqid=0 2024-11-17T15:37:30,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742034_1210 (size=14994) 2024-11-17T15:37:30,766 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:30,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48396 deadline: 1731857910762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:30,766 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:30,766 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:30,766 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:30,877 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:30,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48396 deadline: 1731857910873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:30,878 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:30,878 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:30,878 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:31,083 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:31,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48396 deadline: 1731857911083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:31,084 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:31,084 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:31,084 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:31,159 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:31,163 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411179c5f48f2ffff4c5c8622c4482c9f36a0_34c72dd45fe3d9e6a1558a16ef53eb06 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411179c5f48f2ffff4c5c8622c4482c9f36a0_34c72dd45fe3d9e6a1558a16ef53eb06 2024-11-17T15:37:31,164 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/A/33b502782f0642fab5666eca3e0c0f19, store: [table=TestAcidGuarantees family=A region=34c72dd45fe3d9e6a1558a16ef53eb06] 2024-11-17T15:37:31,165 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/A/33b502782f0642fab5666eca3e0c0f19 is 175, key is test_row_0/A:col10/1731857850744/Put/seqid=0 2024-11-17T15:37:31,168 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742035_1211 (size=39949) 2024-11-17T15:37:31,394 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:31,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48396 deadline: 1731857911393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:31,395 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:31,395 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:31,395 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:31,569 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=352, memsize=55.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/A/33b502782f0642fab5666eca3e0c0f19 2024-11-17T15:37:31,578 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/B/7c0714fc93cf401abdfd90bd9e05fc7d is 50, key is test_row_0/B:col10/1731857850744/Put/seqid=0 2024-11-17T15:37:31,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742036_1212 (size=12301) 2024-11-17T15:37:31,904 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:31,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48396 deadline: 1731857911902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:31,905 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:31,905 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:31,905 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:31,937 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:31,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48428 deadline: 1731857911933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:31,938 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 7a780d55532c,32955,1731857800648 for 'test_row_2' in 34c72dd45fe3d9e6a1558a16ef53eb06 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4200 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T15:37:31,938 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:31,938 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:31,938 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:31,984 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=352 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/B/7c0714fc93cf401abdfd90bd9e05fc7d 2024-11-17T15:37:31,993 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/C/ed857745c8c54cda95e338e224edff14 is 50, key is test_row_0/C:col10/1731857850744/Put/seqid=0 2024-11-17T15:37:31,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742037_1213 (size=12301) 2024-11-17T15:37:32,043 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:32,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48438 deadline: 1731857912043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:32,044 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 7a780d55532c,32955,1731857800648 for 'test_row_1' in 34c72dd45fe3d9e6a1558a16ef53eb06 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4185 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T15:37:32,045 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:32,045 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:32,045 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:32,047 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:32,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48408 deadline: 1731857912043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:32,047 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 7a780d55532c,32955,1731857800648 for 'test_row_1' in 34c72dd45fe3d9e6a1558a16ef53eb06 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4185 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T15:37:32,048 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:32,048 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:32,048 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:32,056 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:32,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48420 deadline: 1731857912053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:32,057 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 7a780d55532c,32955,1731857800648 for 'test_row_1' in 34c72dd45fe3d9e6a1558a16ef53eb06 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4198 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T15:37:32,058 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:32,058 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:32,058 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:32,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-17T15:37:32,123 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T15:37:32,124 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T15:37:32,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] procedure2.ProcedureExecutor(1139): Stored pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees 2024-11-17T15:37:32,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-17T15:37:32,126 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T15:37:32,126 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T15:37:32,126 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T15:37:32,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-17T15:37:32,279 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-17T15:37:32,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. 2024-11-17T15:37:32,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. as already flushing 2024-11-17T15:37:32,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. 2024-11-17T15:37:32,280 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 34c72dd45fe3d9e6a1558a16ef53eb06, NAME => 'TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:37:32,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 34c72dd45fe3d9e6a1558a16ef53eb06, NAME => 'TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:37:32,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 34c72dd45fe3d9e6a1558a16ef53eb06, NAME => 'TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 34c72dd45fe3d9e6a1558a16ef53eb06, NAME => 'TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:37:32,398 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=352 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/C/ed857745c8c54cda95e338e224edff14 2024-11-17T15:37:32,406 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/A/33b502782f0642fab5666eca3e0c0f19 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/33b502782f0642fab5666eca3e0c0f19 2024-11-17T15:37:32,410 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/33b502782f0642fab5666eca3e0c0f19, entries=200, sequenceid=352, filesize=39.0 K 2024-11-17T15:37:32,412 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/B/7c0714fc93cf401abdfd90bd9e05fc7d as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/B/7c0714fc93cf401abdfd90bd9e05fc7d 2024-11-17T15:37:32,416 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/B/7c0714fc93cf401abdfd90bd9e05fc7d, entries=150, sequenceid=352, filesize=12.0 K 2024-11-17T15:37:32,417 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/C/ed857745c8c54cda95e338e224edff14 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/C/ed857745c8c54cda95e338e224edff14 2024-11-17T15:37:32,421 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/C/ed857745c8c54cda95e338e224edff14, entries=150, sequenceid=352, filesize=12.0 K 2024-11-17T15:37:32,422 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=33.54 KB/34350 for 34c72dd45fe3d9e6a1558a16ef53eb06 in 1676ms, sequenceid=352, compaction requested=true 2024-11-17T15:37:32,422 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 34c72dd45fe3d9e6a1558a16ef53eb06: 2024-11-17T15:37:32,422 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 34c72dd45fe3d9e6a1558a16ef53eb06:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T15:37:32,423 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T15:37:32,423 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T15:37:32,423 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 34c72dd45fe3d9e6a1558a16ef53eb06:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T15:37:32,423 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T15:37:32,423 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 34c72dd45fe3d9e6a1558a16ef53eb06:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T15:37:32,423 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T15:37:32,423 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T15:37:32,424 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 134430 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T15:37:32,424 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1541): 34c72dd45fe3d9e6a1558a16ef53eb06/A is initiating minor compaction (all files) 2024-11-17T15:37:32,424 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 34c72dd45fe3d9e6a1558a16ef53eb06/A in TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. 2024-11-17T15:37:32,425 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/da3ea977d6e64552982debb9e49a0656, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/93ca7622c0534237856b58aee1fddeba, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/e4c877389119447d9e92b422daeed890, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/33b502782f0642fab5666eca3e0c0f19] into tmpdir=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp, totalSize=131.3 K 2024-11-17T15:37:32,425 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. 2024-11-17T15:37:32,425 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. files: [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/da3ea977d6e64552982debb9e49a0656, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/93ca7622c0534237856b58aee1fddeba, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/e4c877389119447d9e92b422daeed890, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/33b502782f0642fab5666eca3e0c0f19] 2024-11-17T15:37:32,425 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49920 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T15:37:32,425 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting da3ea977d6e64552982debb9e49a0656, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1731857847620 2024-11-17T15:37:32,425 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1541): 34c72dd45fe3d9e6a1558a16ef53eb06/B is initiating minor compaction (all files) 2024-11-17T15:37:32,425 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 34c72dd45fe3d9e6a1558a16ef53eb06/B in TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. 2024-11-17T15:37:32,425 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/B/f1ae25541f53409dbb5c19fb9d4e82bd, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/B/7cd42ed066324fc4bcf1d037ea6871da, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/B/f3a4d63dcd98420b8bb9808dd58eb23f, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/B/7c0714fc93cf401abdfd90bd9e05fc7d] into tmpdir=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp, totalSize=48.8 K 2024-11-17T15:37:32,426 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 93ca7622c0534237856b58aee1fddeba, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1731857847734 2024-11-17T15:37:32,426 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting f1ae25541f53409dbb5c19fb9d4e82bd, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1731857847620 2024-11-17T15:37:32,426 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting e4c877389119447d9e92b422daeed890, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1731857847852 2024-11-17T15:37:32,426 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting 7cd42ed066324fc4bcf1d037ea6871da, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1731857847734 2024-11-17T15:37:32,427 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 33b502782f0642fab5666eca3e0c0f19, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1731857849558 2024-11-17T15:37:32,427 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting f3a4d63dcd98420b8bb9808dd58eb23f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1731857847852 2024-11-17T15:37:32,427 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting 7c0714fc93cf401abdfd90bd9e05fc7d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1731857849583 2024-11-17T15:37:32,434 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-17T15:37:32,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. 2024-11-17T15:37:32,434 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2902): Flushing 34c72dd45fe3d9e6a1558a16ef53eb06 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-17T15:37:32,439 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=34c72dd45fe3d9e6a1558a16ef53eb06] 2024-11-17T15:37:32,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-17T15:37:32,449 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 34c72dd45fe3d9e6a1558a16ef53eb06#B#compaction#174 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T15:37:32,450 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/B/0bf9270d38104a60afdbd20c19c046d0 is 50, key is test_row_0/B:col10/1731857850744/Put/seqid=0 2024-11-17T15:37:32,451 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241117a56cdc4da6b1440ab79396af50b9aa85_34c72dd45fe3d9e6a1558a16ef53eb06 store=[table=TestAcidGuarantees family=A region=34c72dd45fe3d9e6a1558a16ef53eb06] 2024-11-17T15:37:32,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411174a0c934579de4ecfbf043f5d00c3dbc5_34c72dd45fe3d9e6a1558a16ef53eb06 is 50, key is test_row_0/A:col10/1731857850761/Put/seqid=0 2024-11-17T15:37:32,454 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241117a56cdc4da6b1440ab79396af50b9aa85_34c72dd45fe3d9e6a1558a16ef53eb06, store=[table=TestAcidGuarantees family=A region=34c72dd45fe3d9e6a1558a16ef53eb06] 2024-11-17T15:37:32,454 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117a56cdc4da6b1440ab79396af50b9aa85_34c72dd45fe3d9e6a1558a16ef53eb06 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=34c72dd45fe3d9e6a1558a16ef53eb06] 2024-11-17T15:37:32,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742038_1214 (size=13153) 2024-11-17T15:37:32,460 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742040_1216 (size=4469) 2024-11-17T15:37:32,465 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 34c72dd45fe3d9e6a1558a16ef53eb06#A#compaction#173 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T15:37:32,465 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/A/521a18fea4cb4a11ac5b987109294320 is 175, key is test_row_0/A:col10/1731857850744/Put/seqid=0 2024-11-17T15:37:32,466 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/B/0bf9270d38104a60afdbd20c19c046d0 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/B/0bf9270d38104a60afdbd20c19c046d0 2024-11-17T15:37:32,472 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742041_1217 (size=32107) 2024-11-17T15:37:32,475 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742039_1215 (size=12454) 2024-11-17T15:37:32,480 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/A/521a18fea4cb4a11ac5b987109294320 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/521a18fea4cb4a11ac5b987109294320 2024-11-17T15:37:32,487 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 34c72dd45fe3d9e6a1558a16ef53eb06/A of 34c72dd45fe3d9e6a1558a16ef53eb06 into 521a18fea4cb4a11ac5b987109294320(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T15:37:32,487 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 34c72dd45fe3d9e6a1558a16ef53eb06: 2024-11-17T15:37:32,487 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., storeName=34c72dd45fe3d9e6a1558a16ef53eb06/A, priority=12, startTime=1731857852422; duration=0sec 2024-11-17T15:37:32,487 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T15:37:32,487 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 34c72dd45fe3d9e6a1558a16ef53eb06:A 2024-11-17T15:37:32,487 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T15:37:32,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:32,493 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49886 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T15:37:32,493 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1541): 34c72dd45fe3d9e6a1558a16ef53eb06/C is initiating minor compaction (all files) 2024-11-17T15:37:32,493 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 34c72dd45fe3d9e6a1558a16ef53eb06/C in TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. 2024-11-17T15:37:32,493 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/C/2e128634176d46d9af45f70a8e60e2cc, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/C/595f978bb41141eeb3c944ed33e4fe09, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/C/9de6f2908dbb49c684ca539e76317197, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/C/ed857745c8c54cda95e338e224edff14] into tmpdir=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp, totalSize=48.7 K 2024-11-17T15:37:32,495 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2e128634176d46d9af45f70a8e60e2cc, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1731857847620 2024-11-17T15:37:32,496 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 595f978bb41141eeb3c944ed33e4fe09, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1731857847734 2024-11-17T15:37:32,497 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9de6f2908dbb49c684ca539e76317197, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1731857847852 2024-11-17T15:37:32,498 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting ed857745c8c54cda95e338e224edff14, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1731857849583 2024-11-17T15:37:32,498 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411174a0c934579de4ecfbf043f5d00c3dbc5_34c72dd45fe3d9e6a1558a16ef53eb06 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411174a0c934579de4ecfbf043f5d00c3dbc5_34c72dd45fe3d9e6a1558a16ef53eb06 2024-11-17T15:37:32,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/A/54cec5a21fca4480894de27db868c52a, store: [table=TestAcidGuarantees family=A region=34c72dd45fe3d9e6a1558a16ef53eb06] 2024-11-17T15:37:32,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/A/54cec5a21fca4480894de27db868c52a is 175, key is test_row_0/A:col10/1731857850761/Put/seqid=0 2024-11-17T15:37:32,503 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 34c72dd45fe3d9e6a1558a16ef53eb06/B of 34c72dd45fe3d9e6a1558a16ef53eb06 into 0bf9270d38104a60afdbd20c19c046d0(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T15:37:32,503 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 34c72dd45fe3d9e6a1558a16ef53eb06: 2024-11-17T15:37:32,503 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., storeName=34c72dd45fe3d9e6a1558a16ef53eb06/B, priority=12, startTime=1731857852423; duration=0sec 2024-11-17T15:37:32,503 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T15:37:32,503 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 34c72dd45fe3d9e6a1558a16ef53eb06:B 2024-11-17T15:37:32,511 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 34c72dd45fe3d9e6a1558a16ef53eb06#C#compaction#176 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T15:37:32,511 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/C/efc808e557064429812f04702d8c7ef7 is 50, key is test_row_0/C:col10/1731857850744/Put/seqid=0 2024-11-17T15:37:32,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742042_1218 (size=31255) 2024-11-17T15:37:32,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742043_1219 (size=13119) 2024-11-17T15:37:32,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-17T15:37:32,915 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=360, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/A/54cec5a21fca4480894de27db868c52a 2024-11-17T15:37:32,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/B/ce5c934ad473454798bc98afc7275f37 is 50, key is test_row_0/B:col10/1731857850761/Put/seqid=0 2024-11-17T15:37:32,934 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/C/efc808e557064429812f04702d8c7ef7 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/C/efc808e557064429812f04702d8c7ef7 2024-11-17T15:37:32,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(8855): Flush requested on 34c72dd45fe3d9e6a1558a16ef53eb06 2024-11-17T15:37:32,935 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. as already flushing 2024-11-17T15:37:32,943 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 34c72dd45fe3d9e6a1558a16ef53eb06/C of 34c72dd45fe3d9e6a1558a16ef53eb06 into efc808e557064429812f04702d8c7ef7(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T15:37:32,943 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 34c72dd45fe3d9e6a1558a16ef53eb06: 2024-11-17T15:37:32,943 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., storeName=34c72dd45fe3d9e6a1558a16ef53eb06/C, priority=12, startTime=1731857852423; duration=0sec 2024-11-17T15:37:32,943 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T15:37:32,943 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 34c72dd45fe3d9e6a1558a16ef53eb06:C 2024-11-17T15:37:32,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742044_1220 (size=12301) 2024-11-17T15:37:32,955 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=360 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/B/ce5c934ad473454798bc98afc7275f37 2024-11-17T15:37:32,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/C/f22773f749904fda98c74c6e69aae163 is 50, key is test_row_0/C:col10/1731857850761/Put/seqid=0 2024-11-17T15:37:32,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742045_1221 (size=12301) 2024-11-17T15:37:33,073 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:33,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48396 deadline: 1731857913071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:33,073 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:33,073 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:33,073 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:33,184 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:33,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48396 deadline: 1731857913183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:33,184 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:33,184 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:33,185 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:33,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-17T15:37:33,381 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=360 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/C/f22773f749904fda98c74c6e69aae163 2024-11-17T15:37:33,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/A/54cec5a21fca4480894de27db868c52a as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/54cec5a21fca4480894de27db868c52a 2024-11-17T15:37:33,395 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:33,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48396 deadline: 1731857913393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:33,396 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:33,396 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:33,396 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:33,400 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/54cec5a21fca4480894de27db868c52a, entries=150, sequenceid=360, filesize=30.5 K 2024-11-17T15:37:33,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/B/ce5c934ad473454798bc98afc7275f37 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/B/ce5c934ad473454798bc98afc7275f37 2024-11-17T15:37:33,407 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/B/ce5c934ad473454798bc98afc7275f37, entries=150, sequenceid=360, filesize=12.0 K 2024-11-17T15:37:33,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/C/f22773f749904fda98c74c6e69aae163 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/C/f22773f749904fda98c74c6e69aae163 2024-11-17T15:37:33,418 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/C/f22773f749904fda98c74c6e69aae163, entries=150, sequenceid=360, filesize=12.0 K 2024-11-17T15:37:33,420 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=167.72 KB/171750 for 34c72dd45fe3d9e6a1558a16ef53eb06 in 986ms, sequenceid=360, compaction requested=false 2024-11-17T15:37:33,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2603): Flush status journal for 34c72dd45fe3d9e6a1558a16ef53eb06: 2024-11-17T15:37:33,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. 2024-11-17T15:37:33,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=57 2024-11-17T15:37:33,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.HMaster(4169): Remote procedure done, pid=57 2024-11-17T15:37:33,424 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-11-17T15:37:33,424 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2960 sec 2024-11-17T15:37:33,426 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees in 1.3010 sec 2024-11-17T15:37:33,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(8855): Flush requested on 34c72dd45fe3d9e6a1558a16ef53eb06 2024-11-17T15:37:33,706 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T15:37:33,706 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 34c72dd45fe3d9e6a1558a16ef53eb06 3/3 column families, dataSize=174.43 KB heapSize=457.78 KB 2024-11-17T15:37:33,714 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117fcb995ddc76341698d16ba93cc713415_34c72dd45fe3d9e6a1558a16ef53eb06 is 50, key is test_row_0/A:col10/1731857853025/Put/seqid=0 2024-11-17T15:37:33,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742046_1222 (size=14994) 2024-11-17T15:37:33,720 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:33,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48396 deadline: 1731857913719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:33,720 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:33,720 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:33,720 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:33,825 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:33,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48396 deadline: 1731857913823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:33,826 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:33,826 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:33,826 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:34,033 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:34,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48396 deadline: 1731857914033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:34,033 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:34,034 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:34,034 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:34,118 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,123 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117fcb995ddc76341698d16ba93cc713415_34c72dd45fe3d9e6a1558a16ef53eb06 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117fcb995ddc76341698d16ba93cc713415_34c72dd45fe3d9e6a1558a16ef53eb06 2024-11-17T15:37:34,124 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/A/436f015d949746688f3fd9705f9c76b1, store: [table=TestAcidGuarantees family=A region=34c72dd45fe3d9e6a1558a16ef53eb06] 2024-11-17T15:37:34,124 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/A/436f015d949746688f3fd9705f9c76b1 is 175, key is test_row_0/A:col10/1731857853025/Put/seqid=0 2024-11-17T15:37:34,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742047_1223 (size=39949) 2024-11-17T15:37:34,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-17T15:37:34,273 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T15:37:34,273 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T15:37:34,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] procedure2.ProcedureExecutor(1139): Stored pid=58, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees 2024-11-17T15:37:34,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-17T15:37:34,275 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=58, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T15:37:34,276 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=58, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T15:37:34,276 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T15:37:34,344 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:34,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48396 deadline: 1731857914343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:34,344 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:34,344 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:34,344 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:34,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-17T15:37:34,428 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-17T15:37:34,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. 2024-11-17T15:37:34,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. as already flushing 2024-11-17T15:37:34,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. 2024-11-17T15:37:34,429 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 34c72dd45fe3d9e6a1558a16ef53eb06, NAME => 'TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:37:34,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 34c72dd45fe3d9e6a1558a16ef53eb06, NAME => 'TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:37:34,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 34c72dd45fe3d9e6a1558a16ef53eb06, NAME => 'TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 34c72dd45fe3d9e6a1558a16ef53eb06, NAME => 'TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:37:34,532 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=392, memsize=58.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/A/436f015d949746688f3fd9705f9c76b1 2024-11-17T15:37:34,541 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/B/8ae321c83e3d46db89c0627aa5b374a3 is 50, key is test_row_0/B:col10/1731857853025/Put/seqid=0 2024-11-17T15:37:34,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742048_1224 (size=12301) 2024-11-17T15:37:34,546 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=392 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/B/8ae321c83e3d46db89c0627aa5b374a3 2024-11-17T15:37:34,555 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/C/9fdf92d941f34a6a90823354db37b280 is 50, key is test_row_0/C:col10/1731857853025/Put/seqid=0 2024-11-17T15:37:34,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742049_1225 (size=12301) 2024-11-17T15:37:34,561 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=392 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/C/9fdf92d941f34a6a90823354db37b280 2024-11-17T15:37:34,573 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/A/436f015d949746688f3fd9705f9c76b1 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/436f015d949746688f3fd9705f9c76b1 2024-11-17T15:37:34,578 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/436f015d949746688f3fd9705f9c76b1, entries=200, sequenceid=392, filesize=39.0 K 2024-11-17T15:37:34,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,581 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/B/8ae321c83e3d46db89c0627aa5b374a3 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/B/8ae321c83e3d46db89c0627aa5b374a3 2024-11-17T15:37:34,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,583 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-17T15:37:34,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. 2024-11-17T15:37:34,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. as already flushing 2024-11-17T15:37:34,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. 2024-11-17T15:37:34,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,583 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 34c72dd45fe3d9e6a1558a16ef53eb06, NAME => 'TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:37:34,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 34c72dd45fe3d9e6a1558a16ef53eb06, NAME => 'TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:37:34,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 34c72dd45fe3d9e6a1558a16ef53eb06, NAME => 'TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 34c72dd45fe3d9e6a1558a16ef53eb06, NAME => 'TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:37:34,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,587 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/B/8ae321c83e3d46db89c0627aa5b374a3, entries=150, sequenceid=392, filesize=12.0 K 2024-11-17T15:37:34,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,589 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/C/9fdf92d941f34a6a90823354db37b280 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/C/9fdf92d941f34a6a90823354db37b280 2024-11-17T15:37:34,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-17T15:37:34,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,594 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/C/9fdf92d941f34a6a90823354db37b280, entries=150, sequenceid=392, filesize=12.0 K 2024-11-17T15:37:34,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,595 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~174.43 KB/178620, heapSize ~457.73 KB/468720, currentSize=26.84 KB/27480 for 34c72dd45fe3d9e6a1558a16ef53eb06 in 889ms, sequenceid=392, compaction requested=true 2024-11-17T15:37:34,595 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 34c72dd45fe3d9e6a1558a16ef53eb06: 2024-11-17T15:37:34,595 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 34c72dd45fe3d9e6a1558a16ef53eb06:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T15:37:34,595 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T15:37:34,595 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T15:37:34,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,595 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T15:37:34,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,595 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 34c72dd45fe3d9e6a1558a16ef53eb06:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T15:37:34,595 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T15:37:34,595 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 34c72dd45fe3d9e6a1558a16ef53eb06:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T15:37:34,595 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T15:37:34,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,596 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T15:37:34,596 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1541): 34c72dd45fe3d9e6a1558a16ef53eb06/B is initiating minor compaction (all files) 2024-11-17T15:37:34,596 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 34c72dd45fe3d9e6a1558a16ef53eb06/B in TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. 2024-11-17T15:37:34,597 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/B/0bf9270d38104a60afdbd20c19c046d0, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/B/ce5c934ad473454798bc98afc7275f37, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/B/8ae321c83e3d46db89c0627aa5b374a3] into tmpdir=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp, totalSize=36.9 K 2024-11-17T15:37:34,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,597 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting 0bf9270d38104a60afdbd20c19c046d0, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1731857849583 2024-11-17T15:37:34,597 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103311 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T15:37:34,597 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1541): 34c72dd45fe3d9e6a1558a16ef53eb06/A is initiating minor compaction (all files) 2024-11-17T15:37:34,597 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 34c72dd45fe3d9e6a1558a16ef53eb06/A in TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. 2024-11-17T15:37:34,597 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/521a18fea4cb4a11ac5b987109294320, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/54cec5a21fca4480894de27db868c52a, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/436f015d949746688f3fd9705f9c76b1] into tmpdir=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp, totalSize=100.9 K 2024-11-17T15:37:34,597 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. 2024-11-17T15:37:34,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,597 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. files: [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/521a18fea4cb4a11ac5b987109294320, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/54cec5a21fca4480894de27db868c52a, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/436f015d949746688f3fd9705f9c76b1] 2024-11-17T15:37:34,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,598 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting ce5c934ad473454798bc98afc7275f37, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=360, earliestPutTs=1731857850755 2024-11-17T15:37:34,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,598 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 521a18fea4cb4a11ac5b987109294320, keycount=150, bloomtype=ROW, size=31.4 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1731857849583 2024-11-17T15:37:34,599 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting 8ae321c83e3d46db89c0627aa5b374a3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=392, earliestPutTs=1731857853025 2024-11-17T15:37:34,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,599 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 54cec5a21fca4480894de27db868c52a, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=360, earliestPutTs=1731857850755 2024-11-17T15:37:34,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,600 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 436f015d949746688f3fd9705f9c76b1, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=392, earliestPutTs=1731857853025 2024-11-17T15:37:34,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,607 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=34c72dd45fe3d9e6a1558a16ef53eb06] 2024-11-17T15:37:34,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,612 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 34c72dd45fe3d9e6a1558a16ef53eb06#B#compaction#183 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T15:37:34,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,613 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/B/8a51912fb8a74c3cb4f397b5f7847756 is 50, key is test_row_0/B:col10/1731857853025/Put/seqid=0 2024-11-17T15:37:34,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,617 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241117af63cdcfb32040f39d35c6f9d9b53b5c_34c72dd45fe3d9e6a1558a16ef53eb06 store=[table=TestAcidGuarantees family=A region=34c72dd45fe3d9e6a1558a16ef53eb06] 2024-11-17T15:37:34,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742050_1226 (size=13255) 2024-11-17T15:37:34,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,621 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241117af63cdcfb32040f39d35c6f9d9b53b5c_34c72dd45fe3d9e6a1558a16ef53eb06, store=[table=TestAcidGuarantees family=A region=34c72dd45fe3d9e6a1558a16ef53eb06] 2024-11-17T15:37:34,622 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117af63cdcfb32040f39d35c6f9d9b53b5c_34c72dd45fe3d9e6a1558a16ef53eb06 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=34c72dd45fe3d9e6a1558a16ef53eb06] 2024-11-17T15:37:34,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742051_1227 (size=4469) 2024-11-17T15:37:34,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,737 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-17T15:37:34,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. 2024-11-17T15:37:34,737 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2902): Flushing 34c72dd45fe3d9e6a1558a16ef53eb06 3/3 column families, dataSize=26.84 KB heapSize=71.06 KB 2024-11-17T15:37:34,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411174cff7ab05752490994313c5337d0ff0a_34c72dd45fe3d9e6a1558a16ef53eb06 is 50, key is test_row_1/A:col10/1731857853718/Put/seqid=0 2024-11-17T15:37:34,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,775 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742052_1228 (size=9914) 2024-11-17T15:37:34,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:34,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(8855): Flush requested on 34c72dd45fe3d9e6a1558a16ef53eb06 2024-11-17T15:37:34,875 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. as already flushing 2024-11-17T15:37:34,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-17T15:37:34,986 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:34,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48396 deadline: 1731857914985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:34,986 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:34,987 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:34,987 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:35,027 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/B/8a51912fb8a74c3cb4f397b5f7847756 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/B/8a51912fb8a74c3cb4f397b5f7847756 2024-11-17T15:37:35,033 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 34c72dd45fe3d9e6a1558a16ef53eb06#A#compaction#182 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T15:37:35,033 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/A/37a803ea6eda4916a974451f09dcccdd is 175, key is test_row_0/A:col10/1731857853025/Put/seqid=0 2024-11-17T15:37:35,034 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 34c72dd45fe3d9e6a1558a16ef53eb06/B of 34c72dd45fe3d9e6a1558a16ef53eb06 into 8a51912fb8a74c3cb4f397b5f7847756(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T15:37:35,034 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 34c72dd45fe3d9e6a1558a16ef53eb06: 2024-11-17T15:37:35,034 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., storeName=34c72dd45fe3d9e6a1558a16ef53eb06/B, priority=13, startTime=1731857854595; duration=0sec 2024-11-17T15:37:35,034 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T15:37:35,034 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 34c72dd45fe3d9e6a1558a16ef53eb06:B 2024-11-17T15:37:35,034 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T15:37:35,037 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T15:37:35,037 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1541): 34c72dd45fe3d9e6a1558a16ef53eb06/C is initiating minor compaction (all files) 2024-11-17T15:37:35,037 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 34c72dd45fe3d9e6a1558a16ef53eb06/C in TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. 2024-11-17T15:37:35,037 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/C/efc808e557064429812f04702d8c7ef7, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/C/f22773f749904fda98c74c6e69aae163, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/C/9fdf92d941f34a6a90823354db37b280] into tmpdir=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp, totalSize=36.8 K 2024-11-17T15:37:35,038 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting efc808e557064429812f04702d8c7ef7, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1731857849583 2024-11-17T15:37:35,038 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting f22773f749904fda98c74c6e69aae163, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=360, earliestPutTs=1731857850755 2024-11-17T15:37:35,039 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting 9fdf92d941f34a6a90823354db37b280, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=392, earliestPutTs=1731857853025 2024-11-17T15:37:35,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742053_1229 (size=32209) 2024-11-17T15:37:35,052 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/A/37a803ea6eda4916a974451f09dcccdd as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/37a803ea6eda4916a974451f09dcccdd 2024-11-17T15:37:35,062 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 34c72dd45fe3d9e6a1558a16ef53eb06/A of 34c72dd45fe3d9e6a1558a16ef53eb06 into 37a803ea6eda4916a974451f09dcccdd(size=31.5 K), total size for store is 31.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T15:37:35,062 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 34c72dd45fe3d9e6a1558a16ef53eb06: 2024-11-17T15:37:35,062 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., storeName=34c72dd45fe3d9e6a1558a16ef53eb06/A, priority=13, startTime=1731857854595; duration=0sec 2024-11-17T15:37:35,062 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T15:37:35,062 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 34c72dd45fe3d9e6a1558a16ef53eb06:A 2024-11-17T15:37:35,063 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 34c72dd45fe3d9e6a1558a16ef53eb06#C#compaction#185 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T15:37:35,064 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/C/e7af73895dd54645887cd3d1cb1dffcc is 50, key is test_row_0/C:col10/1731857853025/Put/seqid=0 2024-11-17T15:37:35,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742054_1230 (size=13221) 2024-11-17T15:37:35,094 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:35,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48396 deadline: 1731857915093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:35,095 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:35,095 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:35,095 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:35,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:35,182 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411174cff7ab05752490994313c5337d0ff0a_34c72dd45fe3d9e6a1558a16ef53eb06 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411174cff7ab05752490994313c5337d0ff0a_34c72dd45fe3d9e6a1558a16ef53eb06 2024-11-17T15:37:35,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/A/f71c6db75524446d89ffbee6dc95395c, store: [table=TestAcidGuarantees family=A region=34c72dd45fe3d9e6a1558a16ef53eb06] 2024-11-17T15:37:35,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/A/f71c6db75524446d89ffbee6dc95395c is 175, key is test_row_1/A:col10/1731857853718/Put/seqid=0 2024-11-17T15:37:35,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742055_1231 (size=22561) 2024-11-17T15:37:35,305 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:35,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48396 deadline: 1731857915303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:35,305 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:35,305 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:35,305 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:35,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-17T15:37:35,477 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/C/e7af73895dd54645887cd3d1cb1dffcc as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/C/e7af73895dd54645887cd3d1cb1dffcc 2024-11-17T15:37:35,483 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 34c72dd45fe3d9e6a1558a16ef53eb06/C of 34c72dd45fe3d9e6a1558a16ef53eb06 into e7af73895dd54645887cd3d1cb1dffcc(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T15:37:35,483 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 34c72dd45fe3d9e6a1558a16ef53eb06: 2024-11-17T15:37:35,483 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., storeName=34c72dd45fe3d9e6a1558a16ef53eb06/C, priority=13, startTime=1731857854595; duration=0sec 2024-11-17T15:37:35,483 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T15:37:35,483 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 34c72dd45fe3d9e6a1558a16ef53eb06:C 2024-11-17T15:37:35,592 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=399, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/A/f71c6db75524446d89ffbee6dc95395c 2024-11-17T15:37:35,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/B/93862e74c3574b7c8d04d443dab0f121 is 50, key is test_row_1/B:col10/1731857853718/Put/seqid=0 2024-11-17T15:37:35,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742056_1232 (size=9857) 2024-11-17T15:37:35,608 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=399 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/B/93862e74c3574b7c8d04d443dab0f121 2024-11-17T15:37:35,614 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:35,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48396 deadline: 1731857915613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:35,614 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:35,614 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:35,614 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:35,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/C/f839ecfaf41c439fba4de868e6f158f9 is 50, key is test_row_1/C:col10/1731857853718/Put/seqid=0 2024-11-17T15:37:35,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742057_1233 (size=9857) 2024-11-17T15:37:35,936 INFO [Thread-597 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-597. 2024-11-17T15:37:35,936 DEBUG [Thread-597 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T15:37:35,936 DEBUG [Thread-597 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:37:35,936 DEBUG [Thread-597 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:37:35,936 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T15:37:35,939 INFO [Thread-601 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-601. 2024-11-17T15:37:35,939 DEBUG [Thread-601 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T15:37:35,939 DEBUG [Thread-601 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:37:35,939 DEBUG [Thread-601 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:37:35,939 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T15:37:35,940 INFO [Thread-599 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-599. 2024-11-17T15:37:35,940 DEBUG [Thread-599 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T15:37:35,940 DEBUG [Thread-599 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:37:35,940 DEBUG [Thread-599 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:37:35,940 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T15:37:35,943 INFO [Thread-603 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-603. 2024-11-17T15:37:35,943 DEBUG [Thread-603 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T15:37:35,943 DEBUG [Thread-603 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:37:35,943 DEBUG [Thread-603 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:37:35,943 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T15:37:35,953 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:35,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48428 deadline: 1731857915953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:35,954 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 7a780d55532c,32955,1731857800648 for 'test_row_2' in 34c72dd45fe3d9e6a1558a16ef53eb06 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8216 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T15:37:35,954 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:35,954 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:35,954 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:37:36,021 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=399 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/C/f839ecfaf41c439fba4de868e6f158f9 2024-11-17T15:37:36,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/A/f71c6db75524446d89ffbee6dc95395c as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/f71c6db75524446d89ffbee6dc95395c 2024-11-17T15:37:36,028 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/f71c6db75524446d89ffbee6dc95395c, entries=100, sequenceid=399, filesize=22.0 K 2024-11-17T15:37:36,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/B/93862e74c3574b7c8d04d443dab0f121 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/B/93862e74c3574b7c8d04d443dab0f121 2024-11-17T15:37:36,033 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/B/93862e74c3574b7c8d04d443dab0f121, entries=100, sequenceid=399, filesize=9.6 K 2024-11-17T15:37:36,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/C/f839ecfaf41c439fba4de868e6f158f9 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/C/f839ecfaf41c439fba4de868e6f158f9 2024-11-17T15:37:36,037 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/C/f839ecfaf41c439fba4de868e6f158f9, entries=100, sequenceid=399, filesize=9.6 K 2024-11-17T15:37:36,038 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=174.43 KB/178620 for 34c72dd45fe3d9e6a1558a16ef53eb06 in 1301ms, sequenceid=399, compaction requested=false 2024-11-17T15:37:36,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2603): Flush status journal for 34c72dd45fe3d9e6a1558a16ef53eb06: 2024-11-17T15:37:36,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. 2024-11-17T15:37:36,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=59 2024-11-17T15:37:36,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.HMaster(4169): Remote procedure done, pid=59 2024-11-17T15:37:36,041 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-11-17T15:37:36,041 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7630 sec 2024-11-17T15:37:36,043 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees in 1.7690 sec 2024-11-17T15:37:36,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(8855): Flush requested on 34c72dd45fe3d9e6a1558a16ef53eb06 2024-11-17T15:37:36,064 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T15:37:36,065 INFO [Thread-586 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-586. 2024-11-17T15:37:36,065 INFO [Thread-592 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-592. 2024-11-17T15:37:36,065 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 34c72dd45fe3d9e6a1558a16ef53eb06 3/3 column families, dataSize=187.85 KB heapSize=492.94 KB 2024-11-17T15:37:36,065 DEBUG [Thread-586 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T15:37:36,065 DEBUG [Thread-592 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T15:37:36,065 DEBUG [Thread-586 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:37:36,065 DEBUG [Thread-592 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:37:36,065 DEBUG [Thread-592 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:37:36,065 DEBUG [Thread-586 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:37:36,065 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T15:37:36,065 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T15:37:36,070 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117c75788ef77e14171ae0a5e1ac82c49ec_34c72dd45fe3d9e6a1558a16ef53eb06 is 50, key is test_row_0/A:col10/1731857854973/Put/seqid=0 2024-11-17T15:37:36,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742058_1234 (size=12454) 2024-11-17T15:37:36,074 INFO [Thread-590 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-590. 2024-11-17T15:37:36,074 DEBUG [Thread-590 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T15:37:36,074 DEBUG [Thread-590 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:37:36,075 DEBUG [Thread-590 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:37:36,075 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T15:37:36,124 INFO [Thread-594 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-594. 2024-11-17T15:37:36,124 DEBUG [Thread-594 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T15:37:36,124 DEBUG [Thread-594 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:37:36,124 DEBUG [Thread-594 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:37:36,124 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T15:37:36,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-17T15:37:36,423 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T15:37:36,475 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:36,478 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117c75788ef77e14171ae0a5e1ac82c49ec_34c72dd45fe3d9e6a1558a16ef53eb06 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117c75788ef77e14171ae0a5e1ac82c49ec_34c72dd45fe3d9e6a1558a16ef53eb06 2024-11-17T15:37:36,479 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/A/dc8ffccbefec4052be63c1a2bd686208, store: [table=TestAcidGuarantees family=A region=34c72dd45fe3d9e6a1558a16ef53eb06] 2024-11-17T15:37:36,480 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/A/dc8ffccbefec4052be63c1a2bd686208 is 175, key is test_row_0/A:col10/1731857854973/Put/seqid=0 2024-11-17T15:37:36,483 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742059_1235 (size=31255) 2024-11-17T15:37:36,884 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=433, memsize=62.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/A/dc8ffccbefec4052be63c1a2bd686208 2024-11-17T15:37:36,890 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/B/61a38e27659a454a8cfe0c056d0b090f is 50, key is test_row_0/B:col10/1731857854973/Put/seqid=0 2024-11-17T15:37:36,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742060_1236 (size=12301) 2024-11-17T15:37:37,294 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=62.62 KB at sequenceid=433 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/B/61a38e27659a454a8cfe0c056d0b090f 2024-11-17T15:37:37,301 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/C/c22a87f3fa9047da84e1dcdc1e03da19 is 50, key is test_row_0/C:col10/1731857854973/Put/seqid=0 2024-11-17T15:37:37,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742061_1237 (size=12301) 2024-11-17T15:37:37,705 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=62.62 KB at sequenceid=433 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/C/c22a87f3fa9047da84e1dcdc1e03da19 2024-11-17T15:37:37,709 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/A/dc8ffccbefec4052be63c1a2bd686208 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/dc8ffccbefec4052be63c1a2bd686208 2024-11-17T15:37:37,712 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/dc8ffccbefec4052be63c1a2bd686208, entries=150, sequenceid=433, filesize=30.5 K 2024-11-17T15:37:37,713 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/B/61a38e27659a454a8cfe0c056d0b090f as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/B/61a38e27659a454a8cfe0c056d0b090f 2024-11-17T15:37:37,716 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/B/61a38e27659a454a8cfe0c056d0b090f, entries=150, sequenceid=433, filesize=12.0 K 2024-11-17T15:37:37,717 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/C/c22a87f3fa9047da84e1dcdc1e03da19 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/C/c22a87f3fa9047da84e1dcdc1e03da19 2024-11-17T15:37:37,721 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/C/c22a87f3fa9047da84e1dcdc1e03da19, entries=150, sequenceid=433, filesize=12.0 K 2024-11-17T15:37:37,722 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~187.85 KB/192360, heapSize ~492.89 KB/504720, currentSize=13.42 KB/13740 for 34c72dd45fe3d9e6a1558a16ef53eb06 in 1657ms, sequenceid=433, compaction requested=true 2024-11-17T15:37:37,722 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 34c72dd45fe3d9e6a1558a16ef53eb06: 2024-11-17T15:37:37,722 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 34c72dd45fe3d9e6a1558a16ef53eb06:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T15:37:37,722 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T15:37:37,722 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T15:37:37,722 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 34c72dd45fe3d9e6a1558a16ef53eb06:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T15:37:37,722 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T15:37:37,722 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 34c72dd45fe3d9e6a1558a16ef53eb06:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T15:37:37,722 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T15:37:37,722 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T15:37:37,723 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35413 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T15:37:37,723 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 86025 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T15:37:37,723 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1541): 34c72dd45fe3d9e6a1558a16ef53eb06/B is initiating minor compaction (all files) 2024-11-17T15:37:37,723 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1541): 34c72dd45fe3d9e6a1558a16ef53eb06/A is initiating minor compaction (all files) 2024-11-17T15:37:37,723 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 34c72dd45fe3d9e6a1558a16ef53eb06/B in TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. 2024-11-17T15:37:37,723 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 34c72dd45fe3d9e6a1558a16ef53eb06/A in TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. 2024-11-17T15:37:37,724 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/B/8a51912fb8a74c3cb4f397b5f7847756, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/B/93862e74c3574b7c8d04d443dab0f121, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/B/61a38e27659a454a8cfe0c056d0b090f] into tmpdir=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp, totalSize=34.6 K 2024-11-17T15:37:37,724 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/37a803ea6eda4916a974451f09dcccdd, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/f71c6db75524446d89ffbee6dc95395c, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/dc8ffccbefec4052be63c1a2bd686208] into tmpdir=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp, totalSize=84.0 K 2024-11-17T15:37:37,724 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. 2024-11-17T15:37:37,724 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. files: [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/37a803ea6eda4916a974451f09dcccdd, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/f71c6db75524446d89ffbee6dc95395c, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/dc8ffccbefec4052be63c1a2bd686208] 2024-11-17T15:37:37,724 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting 8a51912fb8a74c3cb4f397b5f7847756, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=392, earliestPutTs=1731857853025 2024-11-17T15:37:37,724 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 37a803ea6eda4916a974451f09dcccdd, keycount=150, bloomtype=ROW, size=31.5 K, encoding=NONE, compression=NONE, seqNum=392, earliestPutTs=1731857853025 2024-11-17T15:37:37,724 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting 93862e74c3574b7c8d04d443dab0f121, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=399, earliestPutTs=1731857853713 2024-11-17T15:37:37,724 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting f71c6db75524446d89ffbee6dc95395c, keycount=100, bloomtype=ROW, size=22.0 K, encoding=NONE, compression=NONE, seqNum=399, earliestPutTs=1731857853713 2024-11-17T15:37:37,724 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting 61a38e27659a454a8cfe0c056d0b090f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=433, earliestPutTs=1731857854973 2024-11-17T15:37:37,725 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting dc8ffccbefec4052be63c1a2bd686208, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=433, earliestPutTs=1731857854973 2024-11-17T15:37:37,732 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 34c72dd45fe3d9e6a1558a16ef53eb06#B#compaction#191 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T15:37:37,733 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/B/2bb3530939ec47c4b11c744dcc816854 is 50, key is test_row_0/B:col10/1731857854973/Put/seqid=0 2024-11-17T15:37:37,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742062_1238 (size=13357) 2024-11-17T15:37:37,740 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=34c72dd45fe3d9e6a1558a16ef53eb06] 2024-11-17T15:37:37,742 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241117c22c5b582bd54240944d2eb19b749f2b_34c72dd45fe3d9e6a1558a16ef53eb06 store=[table=TestAcidGuarantees family=A region=34c72dd45fe3d9e6a1558a16ef53eb06] 2024-11-17T15:37:37,774 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241117c22c5b582bd54240944d2eb19b749f2b_34c72dd45fe3d9e6a1558a16ef53eb06, store=[table=TestAcidGuarantees family=A region=34c72dd45fe3d9e6a1558a16ef53eb06] 2024-11-17T15:37:37,774 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117c22c5b582bd54240944d2eb19b749f2b_34c72dd45fe3d9e6a1558a16ef53eb06 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=34c72dd45fe3d9e6a1558a16ef53eb06] 2024-11-17T15:37:37,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742063_1239 (size=4469) 2024-11-17T15:37:38,143 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/B/2bb3530939ec47c4b11c744dcc816854 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/B/2bb3530939ec47c4b11c744dcc816854 2024-11-17T15:37:38,147 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 34c72dd45fe3d9e6a1558a16ef53eb06/B of 34c72dd45fe3d9e6a1558a16ef53eb06 into 2bb3530939ec47c4b11c744dcc816854(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T15:37:38,147 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 34c72dd45fe3d9e6a1558a16ef53eb06: 2024-11-17T15:37:38,147 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., storeName=34c72dd45fe3d9e6a1558a16ef53eb06/B, priority=13, startTime=1731857857722; duration=0sec 2024-11-17T15:37:38,147 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T15:37:38,147 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 34c72dd45fe3d9e6a1558a16ef53eb06:B 2024-11-17T15:37:38,148 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T15:37:38,148 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35379 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T15:37:38,148 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1541): 34c72dd45fe3d9e6a1558a16ef53eb06/C is initiating minor compaction (all files) 2024-11-17T15:37:38,148 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 34c72dd45fe3d9e6a1558a16ef53eb06/C in TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. 2024-11-17T15:37:38,148 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/C/e7af73895dd54645887cd3d1cb1dffcc, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/C/f839ecfaf41c439fba4de868e6f158f9, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/C/c22a87f3fa9047da84e1dcdc1e03da19] into tmpdir=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp, totalSize=34.5 K 2024-11-17T15:37:38,149 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting e7af73895dd54645887cd3d1cb1dffcc, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=392, earliestPutTs=1731857853025 2024-11-17T15:37:38,149 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting f839ecfaf41c439fba4de868e6f158f9, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=399, earliestPutTs=1731857853713 2024-11-17T15:37:38,149 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting c22a87f3fa9047da84e1dcdc1e03da19, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=433, earliestPutTs=1731857854973 2024-11-17T15:37:38,156 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 34c72dd45fe3d9e6a1558a16ef53eb06#C#compaction#193 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T15:37:38,157 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/C/0d7cd1400add4c679b2e4e3e8fbc94ce is 50, key is test_row_0/C:col10/1731857854973/Put/seqid=0 2024-11-17T15:37:38,160 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742064_1240 (size=13323) 2024-11-17T15:37:38,180 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 34c72dd45fe3d9e6a1558a16ef53eb06#A#compaction#192 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T15:37:38,181 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/A/ff9378b665d3485a831745f5905d939a is 175, key is test_row_0/A:col10/1731857854973/Put/seqid=0 2024-11-17T15:37:38,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742065_1241 (size=32311) 2024-11-17T15:37:38,565 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/C/0d7cd1400add4c679b2e4e3e8fbc94ce as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/C/0d7cd1400add4c679b2e4e3e8fbc94ce 2024-11-17T15:37:38,569 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 34c72dd45fe3d9e6a1558a16ef53eb06/C of 34c72dd45fe3d9e6a1558a16ef53eb06 into 0d7cd1400add4c679b2e4e3e8fbc94ce(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T15:37:38,569 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 34c72dd45fe3d9e6a1558a16ef53eb06: 2024-11-17T15:37:38,570 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., storeName=34c72dd45fe3d9e6a1558a16ef53eb06/C, priority=13, startTime=1731857857722; duration=0sec 2024-11-17T15:37:38,570 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T15:37:38,570 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 34c72dd45fe3d9e6a1558a16ef53eb06:C 2024-11-17T15:37:38,588 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/A/ff9378b665d3485a831745f5905d939a as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/ff9378b665d3485a831745f5905d939a 2024-11-17T15:37:38,592 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 34c72dd45fe3d9e6a1558a16ef53eb06/A of 34c72dd45fe3d9e6a1558a16ef53eb06 into ff9378b665d3485a831745f5905d939a(size=31.6 K), total size for store is 31.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T15:37:38,592 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 34c72dd45fe3d9e6a1558a16ef53eb06: 2024-11-17T15:37:38,593 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., storeName=34c72dd45fe3d9e6a1558a16ef53eb06/A, priority=13, startTime=1731857857722; duration=0sec 2024-11-17T15:37:38,593 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T15:37:38,593 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 34c72dd45fe3d9e6a1558a16ef53eb06:A 2024-11-17T15:37:39,091 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-17T15:37:46,004 INFO [Thread-588 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-588. 2024-11-17T15:37:46,004 DEBUG [Thread-588 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T15:37:46,004 DEBUG [Thread-588 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:37:46,005 DEBUG [Thread-588 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:37:46,005 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T15:37:46,005 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-17T15:37:46,005 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 70 2024-11-17T15:37:46,005 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 48 2024-11-17T15:37:46,005 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 55 2024-11-17T15:37:46,005 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 55 2024-11-17T15:37:46,005 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 112 2024-11-17T15:37:46,005 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-17T15:37:46,005 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3347 2024-11-17T15:37:46,005 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3480 2024-11-17T15:37:46,005 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-17T15:37:46,005 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1461 2024-11-17T15:37:46,005 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4379 rows 2024-11-17T15:37:46,005 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1473 2024-11-17T15:37:46,005 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4414 rows 2024-11-17T15:37:46,005 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-17T15:37:46,005 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-17T15:37:46,005 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:37:46,005 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:37:46,005 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T15:37:46,006 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-17T15:37:46,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] procedure2.ProcedureExecutor(1139): Stored pid=60, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-17T15:37:46,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-17T15:37:46,009 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731857866009"}]},"ts":"1731857866009"} 2024-11-17T15:37:46,010 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-17T15:37:46,011 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-17T15:37:46,011 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-17T15:37:46,012 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=34c72dd45fe3d9e6a1558a16ef53eb06, UNASSIGN}] 2024-11-17T15:37:46,013 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=34c72dd45fe3d9e6a1558a16ef53eb06, UNASSIGN 2024-11-17T15:37:46,014 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=62 updating hbase:meta row=34c72dd45fe3d9e6a1558a16ef53eb06, regionState=CLOSING, regionLocation=7a780d55532c,32955,1731857800648 2024-11-17T15:37:46,015 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=34c72dd45fe3d9e6a1558a16ef53eb06, UNASSIGN because future has completed 2024-11-17T15:37:46,015 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-17T15:37:46,015 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; CloseRegionProcedure 34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648}] 2024-11-17T15:37:46,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-17T15:37:46,167 INFO [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] handler.UnassignRegionHandler(122): Close 34c72dd45fe3d9e6a1558a16ef53eb06 2024-11-17T15:37:46,167 DEBUG [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-17T15:37:46,167 DEBUG [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(1722): Closing 34c72dd45fe3d9e6a1558a16ef53eb06, disabling compactions & flushes 2024-11-17T15:37:46,167 INFO [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. 2024-11-17T15:37:46,168 DEBUG [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. 2024-11-17T15:37:46,168 DEBUG [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. after waiting 0 ms 2024-11-17T15:37:46,168 DEBUG [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. 2024-11-17T15:37:46,168 INFO [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(2902): Flushing 34c72dd45fe3d9e6a1558a16ef53eb06 3/3 column families, dataSize=20.13 KB heapSize=53.48 KB 2024-11-17T15:37:46,175 DEBUG [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117b4012ce7c2c342a78cc76c052fe70097_34c72dd45fe3d9e6a1558a16ef53eb06 is 50, key is test_row_1/A:col10/1731857856123/Put/seqid=0 2024-11-17T15:37:46,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742066_1242 (size=9914) 2024-11-17T15:37:46,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-17T15:37:46,579 DEBUG [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:46,583 INFO [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117b4012ce7c2c342a78cc76c052fe70097_34c72dd45fe3d9e6a1558a16ef53eb06 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117b4012ce7c2c342a78cc76c052fe70097_34c72dd45fe3d9e6a1558a16ef53eb06 2024-11-17T15:37:46,584 DEBUG [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/A/94cd22aa624e4b27b60ced81169eff82, store: [table=TestAcidGuarantees family=A region=34c72dd45fe3d9e6a1558a16ef53eb06] 2024-11-17T15:37:46,585 DEBUG [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/A/94cd22aa624e4b27b60ced81169eff82 is 175, key is test_row_1/A:col10/1731857856123/Put/seqid=0 2024-11-17T15:37:46,588 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742067_1243 (size=22561) 2024-11-17T15:37:46,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-17T15:37:46,989 INFO [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=442, memsize=6.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/A/94cd22aa624e4b27b60ced81169eff82 2024-11-17T15:37:46,995 DEBUG [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/B/65292552b55d47d0bd8442f13e0d05a7 is 50, key is test_row_1/B:col10/1731857856123/Put/seqid=0 2024-11-17T15:37:46,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742068_1244 (size=9857) 2024-11-17T15:37:47,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-17T15:37:47,399 INFO [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=442 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/B/65292552b55d47d0bd8442f13e0d05a7 2024-11-17T15:37:47,406 DEBUG [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/C/62cb2aa9937c4b6c99ff693f0f860c7c is 50, key is test_row_1/C:col10/1731857856123/Put/seqid=0 2024-11-17T15:37:47,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742069_1245 (size=9857) 2024-11-17T15:37:47,810 INFO [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=442 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/C/62cb2aa9937c4b6c99ff693f0f860c7c 2024-11-17T15:37:47,814 DEBUG [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/A/94cd22aa624e4b27b60ced81169eff82 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/94cd22aa624e4b27b60ced81169eff82 2024-11-17T15:37:47,818 INFO [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/94cd22aa624e4b27b60ced81169eff82, entries=100, sequenceid=442, filesize=22.0 K 2024-11-17T15:37:47,818 DEBUG [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/B/65292552b55d47d0bd8442f13e0d05a7 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/B/65292552b55d47d0bd8442f13e0d05a7 2024-11-17T15:37:47,822 INFO [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/B/65292552b55d47d0bd8442f13e0d05a7, entries=100, sequenceid=442, filesize=9.6 K 2024-11-17T15:37:47,823 DEBUG [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/.tmp/C/62cb2aa9937c4b6c99ff693f0f860c7c as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/C/62cb2aa9937c4b6c99ff693f0f860c7c 2024-11-17T15:37:47,826 INFO [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/C/62cb2aa9937c4b6c99ff693f0f860c7c, entries=100, sequenceid=442, filesize=9.6 K 2024-11-17T15:37:47,827 INFO [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=0 B/0 for 34c72dd45fe3d9e6a1558a16ef53eb06 in 1659ms, sequenceid=442, compaction requested=false 2024-11-17T15:37:47,827 DEBUG [StoreCloser-TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/0d694a078a7e47c5bf8e05e16a9ed856, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/62717ca05a3b4d2d8cce6517b08884a4, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/1e36193653634d50a85aef94075acd11, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/1cb34538f39a4907a01461f0c2e76efc, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/c3d4932a92e34b278e3dedfbef004167, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/f6b0cf725d71426aa52d8bfea323007b, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/46801bd954344096976cf0e0ae332a6e, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/b6bfd52673314433b51ea810c706b8cf, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/c23309e3ef4a41aa9d64ca27df001642, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/42d8a488421545ed8912eb775fd9caaa, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/9d4de9bdedda46688e210242dc598bda, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/8eb91122da1e47c983cd56f610f153f3, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/1b9f26eb6f114219be2a34a39a2bec9f, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/1f6ba0fbf2ec4117858a604d48428b0e, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/92c3f89127634f18981173509e4f423c, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/819bfa332f2549c2af5a739a8fad5fdf, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/308900f4496248a285d0daabb50408cd, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/dac104dc4ea04201915bef9a5cb3c8dc, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/1f2b7a1bc19a4e4d97d609e51b76d0ab, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/8d545cfa415f4bb1b137813b2a6c0ac8, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/47eb3c5928e94409b3b2897c3eeedcaa, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/da3ea977d6e64552982debb9e49a0656, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/93ca7622c0534237856b58aee1fddeba, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/e4c877389119447d9e92b422daeed890, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/33b502782f0642fab5666eca3e0c0f19, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/521a18fea4cb4a11ac5b987109294320, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/54cec5a21fca4480894de27db868c52a, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/436f015d949746688f3fd9705f9c76b1, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/37a803ea6eda4916a974451f09dcccdd, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/f71c6db75524446d89ffbee6dc95395c, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/dc8ffccbefec4052be63c1a2bd686208] to archive 2024-11-17T15:37:47,828 DEBUG [StoreCloser-TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-17T15:37:47,830 DEBUG [StoreCloser-TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/0d694a078a7e47c5bf8e05e16a9ed856 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/0d694a078a7e47c5bf8e05e16a9ed856 2024-11-17T15:37:47,831 DEBUG [StoreCloser-TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/62717ca05a3b4d2d8cce6517b08884a4 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/62717ca05a3b4d2d8cce6517b08884a4 2024-11-17T15:37:47,832 DEBUG [StoreCloser-TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/1e36193653634d50a85aef94075acd11 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/1e36193653634d50a85aef94075acd11 2024-11-17T15:37:47,833 DEBUG [StoreCloser-TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/1cb34538f39a4907a01461f0c2e76efc to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/1cb34538f39a4907a01461f0c2e76efc 2024-11-17T15:37:47,834 DEBUG [StoreCloser-TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/c3d4932a92e34b278e3dedfbef004167 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/c3d4932a92e34b278e3dedfbef004167 2024-11-17T15:37:47,835 DEBUG [StoreCloser-TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/f6b0cf725d71426aa52d8bfea323007b to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/f6b0cf725d71426aa52d8bfea323007b 2024-11-17T15:37:47,836 DEBUG [StoreCloser-TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/46801bd954344096976cf0e0ae332a6e to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/46801bd954344096976cf0e0ae332a6e 2024-11-17T15:37:47,837 DEBUG [StoreCloser-TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/b6bfd52673314433b51ea810c706b8cf to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/b6bfd52673314433b51ea810c706b8cf 2024-11-17T15:37:47,838 DEBUG [StoreCloser-TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/c23309e3ef4a41aa9d64ca27df001642 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/c23309e3ef4a41aa9d64ca27df001642 2024-11-17T15:37:47,839 DEBUG [StoreCloser-TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/42d8a488421545ed8912eb775fd9caaa to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/42d8a488421545ed8912eb775fd9caaa 2024-11-17T15:37:47,840 DEBUG [StoreCloser-TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/9d4de9bdedda46688e210242dc598bda to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/9d4de9bdedda46688e210242dc598bda 2024-11-17T15:37:47,841 DEBUG [StoreCloser-TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/8eb91122da1e47c983cd56f610f153f3 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/8eb91122da1e47c983cd56f610f153f3 2024-11-17T15:37:47,842 DEBUG [StoreCloser-TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/1b9f26eb6f114219be2a34a39a2bec9f to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/1b9f26eb6f114219be2a34a39a2bec9f 2024-11-17T15:37:47,843 DEBUG [StoreCloser-TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/1f6ba0fbf2ec4117858a604d48428b0e to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/1f6ba0fbf2ec4117858a604d48428b0e 2024-11-17T15:37:47,844 DEBUG [StoreCloser-TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/92c3f89127634f18981173509e4f423c to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/92c3f89127634f18981173509e4f423c 2024-11-17T15:37:47,845 DEBUG [StoreCloser-TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/819bfa332f2549c2af5a739a8fad5fdf to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/819bfa332f2549c2af5a739a8fad5fdf 2024-11-17T15:37:47,846 DEBUG [StoreCloser-TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/308900f4496248a285d0daabb50408cd to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/308900f4496248a285d0daabb50408cd 2024-11-17T15:37:47,847 DEBUG [StoreCloser-TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/dac104dc4ea04201915bef9a5cb3c8dc to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/dac104dc4ea04201915bef9a5cb3c8dc 2024-11-17T15:37:47,847 DEBUG [StoreCloser-TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/1f2b7a1bc19a4e4d97d609e51b76d0ab to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/1f2b7a1bc19a4e4d97d609e51b76d0ab 2024-11-17T15:37:47,848 DEBUG [StoreCloser-TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/8d545cfa415f4bb1b137813b2a6c0ac8 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/8d545cfa415f4bb1b137813b2a6c0ac8 2024-11-17T15:37:47,849 DEBUG [StoreCloser-TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/47eb3c5928e94409b3b2897c3eeedcaa to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/47eb3c5928e94409b3b2897c3eeedcaa 2024-11-17T15:37:47,850 DEBUG [StoreCloser-TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/da3ea977d6e64552982debb9e49a0656 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/da3ea977d6e64552982debb9e49a0656 2024-11-17T15:37:47,851 DEBUG [StoreCloser-TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/93ca7622c0534237856b58aee1fddeba to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/93ca7622c0534237856b58aee1fddeba 2024-11-17T15:37:47,852 DEBUG [StoreCloser-TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/e4c877389119447d9e92b422daeed890 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/e4c877389119447d9e92b422daeed890 2024-11-17T15:37:47,853 DEBUG [StoreCloser-TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/33b502782f0642fab5666eca3e0c0f19 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/33b502782f0642fab5666eca3e0c0f19 2024-11-17T15:37:47,854 DEBUG [StoreCloser-TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/521a18fea4cb4a11ac5b987109294320 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/521a18fea4cb4a11ac5b987109294320 2024-11-17T15:37:47,855 DEBUG [StoreCloser-TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/54cec5a21fca4480894de27db868c52a to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/54cec5a21fca4480894de27db868c52a 2024-11-17T15:37:47,856 DEBUG [StoreCloser-TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/436f015d949746688f3fd9705f9c76b1 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/436f015d949746688f3fd9705f9c76b1 2024-11-17T15:37:47,857 DEBUG [StoreCloser-TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/37a803ea6eda4916a974451f09dcccdd to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/37a803ea6eda4916a974451f09dcccdd 2024-11-17T15:37:47,858 DEBUG [StoreCloser-TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/f71c6db75524446d89ffbee6dc95395c to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/f71c6db75524446d89ffbee6dc95395c 2024-11-17T15:37:47,859 DEBUG [StoreCloser-TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/dc8ffccbefec4052be63c1a2bd686208 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/dc8ffccbefec4052be63c1a2bd686208 2024-11-17T15:37:47,861 DEBUG [StoreCloser-TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/B/a08c5b3fe16349a2a54e268388f2b474, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/B/78579c536eec4a0199fc0eb798ee7430, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/B/fec8744dafb14d29af8aaa45211efa1f, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/B/c5bb05e8ecc14638bfd65faddb999177, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/B/fb14878cf5c2491f849182859f0b4a53, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/B/35379b7f83784fdf90a10bce4fd4850c, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/B/542cf35da1554b538c155e8530d86d40, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/B/ddf4281b810a4760a3dc29ab840b5225, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/B/4a6775a1914b4a2daa785d91db2c2f3a, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/B/4d02db3f5f28436580fac6958e3bb965, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/B/40acabc06dd6406dbe403479ce937adb, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/B/558f2d07243c42f6802aac8dae181762, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/B/e161e2d4072040a3a62538911f0d5c95, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/B/a2cc53bf68d54bb39edd62cc83d10932, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/B/af2c4bc77ca34aae960590fc332a2a0f, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/B/8a19ca1080d5476e9431e84401a2dc33, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/B/6f8dbf9aa98d41fc9bf4c3a736f65eb4, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/B/e01856a38d7440b6ada86832558eb10a, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/B/441d8e9769f7465aa080fa51197f4614, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/B/5319f67f29c14f43925cda63d7a7e216, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/B/f1ae25541f53409dbb5c19fb9d4e82bd, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/B/f6619f057a24400ba950207261102d13, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/B/7cd42ed066324fc4bcf1d037ea6871da, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/B/f3a4d63dcd98420b8bb9808dd58eb23f, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/B/0bf9270d38104a60afdbd20c19c046d0, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/B/7c0714fc93cf401abdfd90bd9e05fc7d, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/B/ce5c934ad473454798bc98afc7275f37, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/B/8a51912fb8a74c3cb4f397b5f7847756, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/B/8ae321c83e3d46db89c0627aa5b374a3, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/B/93862e74c3574b7c8d04d443dab0f121, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/B/61a38e27659a454a8cfe0c056d0b090f] to archive 2024-11-17T15:37:47,861 DEBUG [StoreCloser-TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-17T15:37:47,863 DEBUG [StoreCloser-TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/B/a08c5b3fe16349a2a54e268388f2b474 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/B/a08c5b3fe16349a2a54e268388f2b474 2024-11-17T15:37:47,864 DEBUG [StoreCloser-TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/B/78579c536eec4a0199fc0eb798ee7430 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/B/78579c536eec4a0199fc0eb798ee7430 2024-11-17T15:37:47,865 DEBUG [StoreCloser-TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/B/fec8744dafb14d29af8aaa45211efa1f to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/B/fec8744dafb14d29af8aaa45211efa1f 2024-11-17T15:37:47,866 DEBUG [StoreCloser-TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/B/c5bb05e8ecc14638bfd65faddb999177 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/B/c5bb05e8ecc14638bfd65faddb999177 2024-11-17T15:37:47,867 DEBUG [StoreCloser-TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/B/fb14878cf5c2491f849182859f0b4a53 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/B/fb14878cf5c2491f849182859f0b4a53 2024-11-17T15:37:47,868 DEBUG [StoreCloser-TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/B/35379b7f83784fdf90a10bce4fd4850c to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/B/35379b7f83784fdf90a10bce4fd4850c 2024-11-17T15:37:47,869 DEBUG [StoreCloser-TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/B/542cf35da1554b538c155e8530d86d40 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/B/542cf35da1554b538c155e8530d86d40 2024-11-17T15:37:47,870 DEBUG [StoreCloser-TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/B/ddf4281b810a4760a3dc29ab840b5225 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/B/ddf4281b810a4760a3dc29ab840b5225 2024-11-17T15:37:47,871 DEBUG [StoreCloser-TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/B/4a6775a1914b4a2daa785d91db2c2f3a to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/B/4a6775a1914b4a2daa785d91db2c2f3a 2024-11-17T15:37:47,872 DEBUG [StoreCloser-TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/B/4d02db3f5f28436580fac6958e3bb965 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/B/4d02db3f5f28436580fac6958e3bb965 2024-11-17T15:37:47,873 DEBUG [StoreCloser-TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/B/40acabc06dd6406dbe403479ce937adb to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/B/40acabc06dd6406dbe403479ce937adb 2024-11-17T15:37:47,873 DEBUG [StoreCloser-TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/B/558f2d07243c42f6802aac8dae181762 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/B/558f2d07243c42f6802aac8dae181762 2024-11-17T15:37:47,874 DEBUG [StoreCloser-TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/B/e161e2d4072040a3a62538911f0d5c95 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/B/e161e2d4072040a3a62538911f0d5c95 2024-11-17T15:37:47,875 DEBUG [StoreCloser-TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/B/a2cc53bf68d54bb39edd62cc83d10932 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/B/a2cc53bf68d54bb39edd62cc83d10932 2024-11-17T15:37:47,876 DEBUG [StoreCloser-TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/B/af2c4bc77ca34aae960590fc332a2a0f to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/B/af2c4bc77ca34aae960590fc332a2a0f 2024-11-17T15:37:47,877 DEBUG [StoreCloser-TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/B/8a19ca1080d5476e9431e84401a2dc33 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/B/8a19ca1080d5476e9431e84401a2dc33 2024-11-17T15:37:47,878 DEBUG [StoreCloser-TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/B/6f8dbf9aa98d41fc9bf4c3a736f65eb4 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/B/6f8dbf9aa98d41fc9bf4c3a736f65eb4 2024-11-17T15:37:47,879 DEBUG [StoreCloser-TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/B/e01856a38d7440b6ada86832558eb10a to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/B/e01856a38d7440b6ada86832558eb10a 2024-11-17T15:37:47,880 DEBUG [StoreCloser-TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/B/441d8e9769f7465aa080fa51197f4614 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/B/441d8e9769f7465aa080fa51197f4614 2024-11-17T15:37:47,881 DEBUG [StoreCloser-TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/B/5319f67f29c14f43925cda63d7a7e216 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/B/5319f67f29c14f43925cda63d7a7e216 2024-11-17T15:37:47,881 DEBUG [StoreCloser-TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/B/f1ae25541f53409dbb5c19fb9d4e82bd to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/B/f1ae25541f53409dbb5c19fb9d4e82bd 2024-11-17T15:37:47,882 DEBUG [StoreCloser-TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/B/f6619f057a24400ba950207261102d13 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/B/f6619f057a24400ba950207261102d13 2024-11-17T15:37:47,883 DEBUG [StoreCloser-TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/B/7cd42ed066324fc4bcf1d037ea6871da to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/B/7cd42ed066324fc4bcf1d037ea6871da 2024-11-17T15:37:47,884 DEBUG [StoreCloser-TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/B/f3a4d63dcd98420b8bb9808dd58eb23f to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/B/f3a4d63dcd98420b8bb9808dd58eb23f 2024-11-17T15:37:47,885 DEBUG [StoreCloser-TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/B/0bf9270d38104a60afdbd20c19c046d0 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/B/0bf9270d38104a60afdbd20c19c046d0 2024-11-17T15:37:47,886 DEBUG [StoreCloser-TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/B/7c0714fc93cf401abdfd90bd9e05fc7d to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/B/7c0714fc93cf401abdfd90bd9e05fc7d 2024-11-17T15:37:47,887 DEBUG [StoreCloser-TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/B/ce5c934ad473454798bc98afc7275f37 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/B/ce5c934ad473454798bc98afc7275f37 2024-11-17T15:37:47,887 DEBUG [StoreCloser-TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/B/8a51912fb8a74c3cb4f397b5f7847756 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/B/8a51912fb8a74c3cb4f397b5f7847756 2024-11-17T15:37:47,888 DEBUG [StoreCloser-TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/B/8ae321c83e3d46db89c0627aa5b374a3 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/B/8ae321c83e3d46db89c0627aa5b374a3 2024-11-17T15:37:47,889 DEBUG [StoreCloser-TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/B/93862e74c3574b7c8d04d443dab0f121 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/B/93862e74c3574b7c8d04d443dab0f121 2024-11-17T15:37:47,890 DEBUG [StoreCloser-TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/B/61a38e27659a454a8cfe0c056d0b090f to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/B/61a38e27659a454a8cfe0c056d0b090f 2024-11-17T15:37:47,892 DEBUG [StoreCloser-TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/C/812b2c151b4a43569136b3f63286e434, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/C/aa994d1e00904e3380146931b98a05e6, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/C/3682a54890f34de09beaa648516b5f36, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/C/f0d3442a16ba4f339f2f4bfb8141ba32, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/C/7b2d694a9e79450585cbd56602818b68, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/C/a8935481cbc74a389c98c34b0650c8d3, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/C/a792eefe4ce54c548f3fb56f254f0321, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/C/6caead97ba93432eb1f4ab8ffaa756d4, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/C/7f59a4ccad1543c4a2e67a41e77bce52, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/C/adb4f2cd788b4cf085e9c464a8225a1d, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/C/3821e93ba88246ad9dc47e96bc7763dd, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/C/d5e9c4855b0d4fe8aed377e03a8972e2, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/C/b8f35c4400a44600b787d03d95d309d5, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/C/99c73440b67a416d9cf345e797bcda0a, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/C/a84beb7952024cc38e7c159a689a0864, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/C/72d4f83ff53e4106ac571c9507fa64fb, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/C/3201087044614829893329419faa4487, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/C/74e7ea14e44b49308a1750d48a1b8f2d, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/C/dc79b2f01ff542fca414e2d9754bba08, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/C/2e128634176d46d9af45f70a8e60e2cc, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/C/4cbd51af63d84631a34bc1c8ff659f16, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/C/595f978bb41141eeb3c944ed33e4fe09, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/C/9de6f2908dbb49c684ca539e76317197, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/C/efc808e557064429812f04702d8c7ef7, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/C/ed857745c8c54cda95e338e224edff14, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/C/f22773f749904fda98c74c6e69aae163, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/C/e7af73895dd54645887cd3d1cb1dffcc, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/C/9fdf92d941f34a6a90823354db37b280, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/C/f839ecfaf41c439fba4de868e6f158f9, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/C/c22a87f3fa9047da84e1dcdc1e03da19] to archive 2024-11-17T15:37:47,893 DEBUG [StoreCloser-TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-17T15:37:47,894 DEBUG [StoreCloser-TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/C/812b2c151b4a43569136b3f63286e434 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/C/812b2c151b4a43569136b3f63286e434 2024-11-17T15:37:47,895 DEBUG [StoreCloser-TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/C/aa994d1e00904e3380146931b98a05e6 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/C/aa994d1e00904e3380146931b98a05e6 2024-11-17T15:37:47,896 DEBUG [StoreCloser-TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/C/3682a54890f34de09beaa648516b5f36 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/C/3682a54890f34de09beaa648516b5f36 2024-11-17T15:37:47,897 DEBUG [StoreCloser-TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/C/f0d3442a16ba4f339f2f4bfb8141ba32 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/C/f0d3442a16ba4f339f2f4bfb8141ba32 2024-11-17T15:37:47,897 DEBUG [StoreCloser-TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/C/7b2d694a9e79450585cbd56602818b68 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/C/7b2d694a9e79450585cbd56602818b68 2024-11-17T15:37:47,898 DEBUG [StoreCloser-TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/C/a8935481cbc74a389c98c34b0650c8d3 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/C/a8935481cbc74a389c98c34b0650c8d3 2024-11-17T15:37:47,899 DEBUG [StoreCloser-TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/C/a792eefe4ce54c548f3fb56f254f0321 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/C/a792eefe4ce54c548f3fb56f254f0321 2024-11-17T15:37:47,900 DEBUG [StoreCloser-TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/C/6caead97ba93432eb1f4ab8ffaa756d4 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/C/6caead97ba93432eb1f4ab8ffaa756d4 2024-11-17T15:37:47,901 DEBUG [StoreCloser-TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/C/7f59a4ccad1543c4a2e67a41e77bce52 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/C/7f59a4ccad1543c4a2e67a41e77bce52 2024-11-17T15:37:47,902 DEBUG [StoreCloser-TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/C/adb4f2cd788b4cf085e9c464a8225a1d to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/C/adb4f2cd788b4cf085e9c464a8225a1d 2024-11-17T15:37:47,903 DEBUG [StoreCloser-TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/C/3821e93ba88246ad9dc47e96bc7763dd to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/C/3821e93ba88246ad9dc47e96bc7763dd 2024-11-17T15:37:47,904 DEBUG [StoreCloser-TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/C/d5e9c4855b0d4fe8aed377e03a8972e2 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/C/d5e9c4855b0d4fe8aed377e03a8972e2 2024-11-17T15:37:47,905 DEBUG [StoreCloser-TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/C/b8f35c4400a44600b787d03d95d309d5 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/C/b8f35c4400a44600b787d03d95d309d5 2024-11-17T15:37:47,906 DEBUG [StoreCloser-TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/C/99c73440b67a416d9cf345e797bcda0a to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/C/99c73440b67a416d9cf345e797bcda0a 2024-11-17T15:37:47,907 DEBUG [StoreCloser-TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/C/a84beb7952024cc38e7c159a689a0864 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/C/a84beb7952024cc38e7c159a689a0864 2024-11-17T15:37:47,908 DEBUG [StoreCloser-TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/C/72d4f83ff53e4106ac571c9507fa64fb to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/C/72d4f83ff53e4106ac571c9507fa64fb 2024-11-17T15:37:47,909 DEBUG [StoreCloser-TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/C/3201087044614829893329419faa4487 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/C/3201087044614829893329419faa4487 2024-11-17T15:37:47,909 DEBUG [StoreCloser-TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/C/74e7ea14e44b49308a1750d48a1b8f2d to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/C/74e7ea14e44b49308a1750d48a1b8f2d 2024-11-17T15:37:47,910 DEBUG [StoreCloser-TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/C/dc79b2f01ff542fca414e2d9754bba08 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/C/dc79b2f01ff542fca414e2d9754bba08 2024-11-17T15:37:47,911 DEBUG [StoreCloser-TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/C/2e128634176d46d9af45f70a8e60e2cc to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/C/2e128634176d46d9af45f70a8e60e2cc 2024-11-17T15:37:47,912 DEBUG [StoreCloser-TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/C/4cbd51af63d84631a34bc1c8ff659f16 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/C/4cbd51af63d84631a34bc1c8ff659f16 2024-11-17T15:37:47,913 DEBUG [StoreCloser-TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/C/595f978bb41141eeb3c944ed33e4fe09 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/C/595f978bb41141eeb3c944ed33e4fe09 2024-11-17T15:37:47,914 DEBUG [StoreCloser-TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/C/9de6f2908dbb49c684ca539e76317197 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/C/9de6f2908dbb49c684ca539e76317197 2024-11-17T15:37:47,916 DEBUG [StoreCloser-TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/C/efc808e557064429812f04702d8c7ef7 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/C/efc808e557064429812f04702d8c7ef7 2024-11-17T15:37:47,917 DEBUG [StoreCloser-TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/C/ed857745c8c54cda95e338e224edff14 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/C/ed857745c8c54cda95e338e224edff14 2024-11-17T15:37:47,917 DEBUG [StoreCloser-TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/C/f22773f749904fda98c74c6e69aae163 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/C/f22773f749904fda98c74c6e69aae163 2024-11-17T15:37:47,918 DEBUG [StoreCloser-TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/C/e7af73895dd54645887cd3d1cb1dffcc to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/C/e7af73895dd54645887cd3d1cb1dffcc 2024-11-17T15:37:47,920 DEBUG [StoreCloser-TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/C/9fdf92d941f34a6a90823354db37b280 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/C/9fdf92d941f34a6a90823354db37b280 2024-11-17T15:37:47,921 DEBUG [StoreCloser-TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/C/f839ecfaf41c439fba4de868e6f158f9 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/C/f839ecfaf41c439fba4de868e6f158f9 2024-11-17T15:37:47,922 DEBUG [StoreCloser-TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/C/c22a87f3fa9047da84e1dcdc1e03da19 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/C/c22a87f3fa9047da84e1dcdc1e03da19 2024-11-17T15:37:47,926 DEBUG [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/recovered.edits/445.seqid, newMaxSeqId=445, maxSeqId=4 2024-11-17T15:37:47,927 INFO [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. 2024-11-17T15:37:47,927 DEBUG [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(1676): Region close journal for 34c72dd45fe3d9e6a1558a16ef53eb06: Waiting for close lock at 1731857866167Running coprocessor pre-close hooks at 1731857866167Disabling compacts and flushes for region at 1731857866167Disabling writes for close at 1731857866168 (+1 ms)Obtaining lock to block concurrent updates at 1731857866168Preparing flush snapshotting stores in 34c72dd45fe3d9e6a1558a16ef53eb06 at 1731857866168Finished memstore snapshotting TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=54720, getOffHeapSize=0, getCellsCount=450 at 1731857866168Flushing stores of TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06. at 1731857866169 (+1 ms)Flushing 34c72dd45fe3d9e6a1558a16ef53eb06/A: creating writer at 1731857866169Flushing mob file 34c72dd45fe3d9e6a1558a16ef53eb06/A: appending metadata at 1731857866174 (+5 ms)Flushing mob file 34c72dd45fe3d9e6a1558a16ef53eb06/A: closing flushed file at 1731857866174Flushing 34c72dd45fe3d9e6a1558a16ef53eb06/A: appending metadata at 1731857866584 (+410 ms)Flushing 34c72dd45fe3d9e6a1558a16ef53eb06/A: closing flushed file at 1731857866584Flushing 34c72dd45fe3d9e6a1558a16ef53eb06/B: creating writer at 1731857866992 (+408 ms)Flushing 34c72dd45fe3d9e6a1558a16ef53eb06/B: appending metadata at 1731857866995 (+3 ms)Flushing 34c72dd45fe3d9e6a1558a16ef53eb06/B: closing flushed file at 1731857866995Flushing 34c72dd45fe3d9e6a1558a16ef53eb06/C: creating writer at 1731857867403 (+408 ms)Flushing 34c72dd45fe3d9e6a1558a16ef53eb06/C: appending metadata at 1731857867405 (+2 ms)Flushing 34c72dd45fe3d9e6a1558a16ef53eb06/C: closing flushed file at 1731857867405Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@30c0a63b: reopening flushed file at 1731857867813 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@76a7bc58: reopening flushed file at 1731857867818 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@14537818: reopening flushed file at 1731857867822 (+4 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=0 B/0 for 34c72dd45fe3d9e6a1558a16ef53eb06 in 1659ms, sequenceid=442, compaction requested=false at 1731857867827 (+5 ms)Writing region close event to WAL at 1731857867923 (+96 ms)Running coprocessor post-close hooks at 1731857867927 (+4 ms)Closed at 1731857867927 2024-11-17T15:37:47,929 INFO [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] handler.UnassignRegionHandler(157): Closed 34c72dd45fe3d9e6a1558a16ef53eb06 2024-11-17T15:37:47,929 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=62 updating hbase:meta row=34c72dd45fe3d9e6a1558a16ef53eb06, regionState=CLOSED 2024-11-17T15:37:47,931 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=63, ppid=62, state=RUNNABLE, hasLock=false; CloseRegionProcedure 34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 because future has completed 2024-11-17T15:37:47,933 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-17T15:37:47,933 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; CloseRegionProcedure 34c72dd45fe3d9e6a1558a16ef53eb06, server=7a780d55532c,32955,1731857800648 in 1.9160 sec 2024-11-17T15:37:47,934 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=62, resume processing ppid=61 2024-11-17T15:37:47,934 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, ppid=61, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=34c72dd45fe3d9e6a1558a16ef53eb06, UNASSIGN in 1.9210 sec 2024-11-17T15:37:47,937 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-17T15:37:47,937 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9240 sec 2024-11-17T15:37:47,938 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731857867938"}]},"ts":"1731857867938"} 2024-11-17T15:37:47,939 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-17T15:37:47,939 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-17T15:37:47,940 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9330 sec 2024-11-17T15:37:48,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-17T15:37:48,153 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-17T15:37:48,153 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-17T15:37:48,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] procedure2.ProcedureExecutor(1139): Stored pid=64, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T15:37:48,155 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=64, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T15:37:48,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-17T15:37:48,156 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=64, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T15:37:48,158 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06 2024-11-17T15:37:48,160 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A, FileablePath, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/B, FileablePath, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/C, FileablePath, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/recovered.edits] 2024-11-17T15:37:48,162 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/94cd22aa624e4b27b60ced81169eff82 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/94cd22aa624e4b27b60ced81169eff82 2024-11-17T15:37:48,163 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/ff9378b665d3485a831745f5905d939a to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/A/ff9378b665d3485a831745f5905d939a 2024-11-17T15:37:48,165 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/B/2bb3530939ec47c4b11c744dcc816854 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/B/2bb3530939ec47c4b11c744dcc816854 2024-11-17T15:37:48,166 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/B/65292552b55d47d0bd8442f13e0d05a7 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/B/65292552b55d47d0bd8442f13e0d05a7 2024-11-17T15:37:48,168 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/C/0d7cd1400add4c679b2e4e3e8fbc94ce to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/C/0d7cd1400add4c679b2e4e3e8fbc94ce 2024-11-17T15:37:48,169 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/C/62cb2aa9937c4b6c99ff693f0f860c7c to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/C/62cb2aa9937c4b6c99ff693f0f860c7c 2024-11-17T15:37:48,172 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/recovered.edits/445.seqid to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06/recovered.edits/445.seqid 2024-11-17T15:37:48,173 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/34c72dd45fe3d9e6a1558a16ef53eb06 2024-11-17T15:37:48,173 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-17T15:37:48,173 DEBUG [PEWorker-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-17T15:37:48,174 DEBUG [PEWorker-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-17T15:37:48,177 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111700b015045823475784c0167ef6f9d141_34c72dd45fe3d9e6a1558a16ef53eb06 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111700b015045823475784c0167ef6f9d141_34c72dd45fe3d9e6a1558a16ef53eb06 2024-11-17T15:37:48,178 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411170860d2049ab943b4b091152722f08ad0_34c72dd45fe3d9e6a1558a16ef53eb06 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411170860d2049ab943b4b091152722f08ad0_34c72dd45fe3d9e6a1558a16ef53eb06 2024-11-17T15:37:48,179 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111709e98b3c6568449f8144b896d2e5bc20_34c72dd45fe3d9e6a1558a16ef53eb06 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111709e98b3c6568449f8144b896d2e5bc20_34c72dd45fe3d9e6a1558a16ef53eb06 2024-11-17T15:37:48,180 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411171465ac8db9ef4f9784468a5b621ed099_34c72dd45fe3d9e6a1558a16ef53eb06 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411171465ac8db9ef4f9784468a5b621ed099_34c72dd45fe3d9e6a1558a16ef53eb06 2024-11-17T15:37:48,181 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411173f22668ee10346949d5fda6524672a93_34c72dd45fe3d9e6a1558a16ef53eb06 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411173f22668ee10346949d5fda6524672a93_34c72dd45fe3d9e6a1558a16ef53eb06 2024-11-17T15:37:48,182 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111742437ab90ecd4ab0b4e46fe4b0612c1b_34c72dd45fe3d9e6a1558a16ef53eb06 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111742437ab90ecd4ab0b4e46fe4b0612c1b_34c72dd45fe3d9e6a1558a16ef53eb06 2024-11-17T15:37:48,184 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411174a0c934579de4ecfbf043f5d00c3dbc5_34c72dd45fe3d9e6a1558a16ef53eb06 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411174a0c934579de4ecfbf043f5d00c3dbc5_34c72dd45fe3d9e6a1558a16ef53eb06 2024-11-17T15:37:48,185 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411174cff7ab05752490994313c5337d0ff0a_34c72dd45fe3d9e6a1558a16ef53eb06 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411174cff7ab05752490994313c5337d0ff0a_34c72dd45fe3d9e6a1558a16ef53eb06 2024-11-17T15:37:48,185 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411174e968bdcd9674cee844a34e18efab779_34c72dd45fe3d9e6a1558a16ef53eb06 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411174e968bdcd9674cee844a34e18efab779_34c72dd45fe3d9e6a1558a16ef53eb06 2024-11-17T15:37:48,186 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411175ea7656ae19c4c82bc00b23a4044f4cd_34c72dd45fe3d9e6a1558a16ef53eb06 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411175ea7656ae19c4c82bc00b23a4044f4cd_34c72dd45fe3d9e6a1558a16ef53eb06 2024-11-17T15:37:48,187 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111779f271a2368f42f3a0d5992831149892_34c72dd45fe3d9e6a1558a16ef53eb06 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111779f271a2368f42f3a0d5992831149892_34c72dd45fe3d9e6a1558a16ef53eb06 2024-11-17T15:37:48,188 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411177c90f771257d495f8625cad85d6ded06_34c72dd45fe3d9e6a1558a16ef53eb06 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411177c90f771257d495f8625cad85d6ded06_34c72dd45fe3d9e6a1558a16ef53eb06 2024-11-17T15:37:48,189 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111782434585db8044cb9a8cef2199d06bc5_34c72dd45fe3d9e6a1558a16ef53eb06 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111782434585db8044cb9a8cef2199d06bc5_34c72dd45fe3d9e6a1558a16ef53eb06 2024-11-17T15:37:48,190 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111784b26e095b0c4ece98c460c0b41eba67_34c72dd45fe3d9e6a1558a16ef53eb06 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111784b26e095b0c4ece98c460c0b41eba67_34c72dd45fe3d9e6a1558a16ef53eb06 2024-11-17T15:37:48,191 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111793bb92ca7ee74f36ad8ff409d7766264_34c72dd45fe3d9e6a1558a16ef53eb06 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111793bb92ca7ee74f36ad8ff409d7766264_34c72dd45fe3d9e6a1558a16ef53eb06 2024-11-17T15:37:48,192 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411179c5f48f2ffff4c5c8622c4482c9f36a0_34c72dd45fe3d9e6a1558a16ef53eb06 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411179c5f48f2ffff4c5c8622c4482c9f36a0_34c72dd45fe3d9e6a1558a16ef53eb06 2024-11-17T15:37:48,193 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117b4012ce7c2c342a78cc76c052fe70097_34c72dd45fe3d9e6a1558a16ef53eb06 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117b4012ce7c2c342a78cc76c052fe70097_34c72dd45fe3d9e6a1558a16ef53eb06 2024-11-17T15:37:48,194 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117c75788ef77e14171ae0a5e1ac82c49ec_34c72dd45fe3d9e6a1558a16ef53eb06 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117c75788ef77e14171ae0a5e1ac82c49ec_34c72dd45fe3d9e6a1558a16ef53eb06 2024-11-17T15:37:48,195 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117ef2d17d9058548a690f179632236993a_34c72dd45fe3d9e6a1558a16ef53eb06 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117ef2d17d9058548a690f179632236993a_34c72dd45fe3d9e6a1558a16ef53eb06 2024-11-17T15:37:48,196 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117f2138863267342a08d6776b42ff12576_34c72dd45fe3d9e6a1558a16ef53eb06 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117f2138863267342a08d6776b42ff12576_34c72dd45fe3d9e6a1558a16ef53eb06 2024-11-17T15:37:48,197 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117f3bc7e29ee9d48518f1015ddb65342f8_34c72dd45fe3d9e6a1558a16ef53eb06 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117f3bc7e29ee9d48518f1015ddb65342f8_34c72dd45fe3d9e6a1558a16ef53eb06 2024-11-17T15:37:48,198 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117fcb995ddc76341698d16ba93cc713415_34c72dd45fe3d9e6a1558a16ef53eb06 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117fcb995ddc76341698d16ba93cc713415_34c72dd45fe3d9e6a1558a16ef53eb06 2024-11-17T15:37:48,199 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117fd9629689cac4af480f7869ab06c983d_34c72dd45fe3d9e6a1558a16ef53eb06 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117fd9629689cac4af480f7869ab06c983d_34c72dd45fe3d9e6a1558a16ef53eb06 2024-11-17T15:37:48,199 DEBUG [PEWorker-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-17T15:37:48,201 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=64, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T15:37:48,204 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-17T15:37:48,206 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-17T15:37:48,207 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=64, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T15:37:48,207 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-17T15:37:48,208 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731857868207"}]},"ts":"9223372036854775807"} 2024-11-17T15:37:48,210 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-17T15:37:48,210 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 34c72dd45fe3d9e6a1558a16ef53eb06, NAME => 'TestAcidGuarantees,,1731857832783.34c72dd45fe3d9e6a1558a16ef53eb06.', STARTKEY => '', ENDKEY => ''}] 2024-11-17T15:37:48,210 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-17T15:37:48,210 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731857868210"}]},"ts":"9223372036854775807"} 2024-11-17T15:37:48,211 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-17T15:37:48,212 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=64, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T15:37:48,213 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 59 msec 2024-11-17T15:37:48,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-17T15:37:48,264 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-17T15:37:48,264 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-17T15:37:48,273 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobMixedAtomicity Thread=238 (was 238), OpenFileDescriptor=449 (was 453), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=371 (was 437), ProcessCount=11 (was 11), AvailableMemoryMB=3221 (was 3341) 2024-11-17T15:37:48,283 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testGetAtomicity Thread=238, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=371, ProcessCount=11, AvailableMemoryMB=3219 2024-11-17T15:37:48,285 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-17T15:37:48,285 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-17T15:37:48,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] procedure2.ProcedureExecutor(1139): Stored pid=65, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-17T15:37:48,287 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=65, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-17T15:37:48,287 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:48,287 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 65 2024-11-17T15:37:48,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=65 2024-11-17T15:37:48,288 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=65, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-17T15:37:48,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742070_1246 (size=959) 2024-11-17T15:37:48,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=65 2024-11-17T15:37:48,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=65 2024-11-17T15:37:48,695 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 5b1940a26f6b6e92cc2f11a0e1deef31, NAME => 'TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206 2024-11-17T15:37:48,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742071_1247 (size=53) 2024-11-17T15:37:48,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=65 2024-11-17T15:37:49,101 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T15:37:49,101 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 5b1940a26f6b6e92cc2f11a0e1deef31, disabling compactions & flushes 2024-11-17T15:37:49,101 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31. 2024-11-17T15:37:49,101 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31. 2024-11-17T15:37:49,101 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31. after waiting 0 ms 2024-11-17T15:37:49,102 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31. 2024-11-17T15:37:49,102 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31. 2024-11-17T15:37:49,102 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 5b1940a26f6b6e92cc2f11a0e1deef31: Waiting for close lock at 1731857869101Disabling compacts and flushes for region at 1731857869101Disabling writes for close at 1731857869101Writing region close event to WAL at 1731857869102 (+1 ms)Closed at 1731857869102 2024-11-17T15:37:49,103 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=65, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-17T15:37:49,103 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731857869103"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731857869103"}]},"ts":"1731857869103"} 2024-11-17T15:37:49,104 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-17T15:37:49,105 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=65, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-17T15:37:49,105 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731857869105"}]},"ts":"1731857869105"} 2024-11-17T15:37:49,106 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-17T15:37:49,107 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=66, ppid=65, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5b1940a26f6b6e92cc2f11a0e1deef31, ASSIGN}] 2024-11-17T15:37:49,107 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=65, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5b1940a26f6b6e92cc2f11a0e1deef31, ASSIGN 2024-11-17T15:37:49,108 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=66, ppid=65, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5b1940a26f6b6e92cc2f11a0e1deef31, ASSIGN; state=OFFLINE, location=7a780d55532c,32955,1731857800648; forceNewPlan=false, retain=false 2024-11-17T15:37:49,259 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=66 updating hbase:meta row=5b1940a26f6b6e92cc2f11a0e1deef31, regionState=OPENING, regionLocation=7a780d55532c,32955,1731857800648 2024-11-17T15:37:49,260 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=66, ppid=65, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5b1940a26f6b6e92cc2f11a0e1deef31, ASSIGN because future has completed 2024-11-17T15:37:49,260 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648}] 2024-11-17T15:37:49,415 INFO [RS_OPEN_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31. 2024-11-17T15:37:49,415 DEBUG [RS_OPEN_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(7752): Opening region: {ENCODED => 5b1940a26f6b6e92cc2f11a0e1deef31, NAME => 'TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.', STARTKEY => '', ENDKEY => ''} 2024-11-17T15:37:49,415 DEBUG [RS_OPEN_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 5b1940a26f6b6e92cc2f11a0e1deef31 2024-11-17T15:37:49,415 DEBUG [RS_OPEN_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T15:37:49,415 DEBUG [RS_OPEN_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(7794): checking encryption for 5b1940a26f6b6e92cc2f11a0e1deef31 2024-11-17T15:37:49,415 DEBUG [RS_OPEN_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(7797): checking classloading for 5b1940a26f6b6e92cc2f11a0e1deef31 2024-11-17T15:37:49,416 INFO [StoreOpener-5b1940a26f6b6e92cc2f11a0e1deef31-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 5b1940a26f6b6e92cc2f11a0e1deef31 2024-11-17T15:37:49,417 INFO [StoreOpener-5b1940a26f6b6e92cc2f11a0e1deef31-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5b1940a26f6b6e92cc2f11a0e1deef31 columnFamilyName A 2024-11-17T15:37:49,417 DEBUG [StoreOpener-5b1940a26f6b6e92cc2f11a0e1deef31-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:49,418 INFO [StoreOpener-5b1940a26f6b6e92cc2f11a0e1deef31-1 {}] regionserver.HStore(327): Store=5b1940a26f6b6e92cc2f11a0e1deef31/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T15:37:49,418 INFO [StoreOpener-5b1940a26f6b6e92cc2f11a0e1deef31-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 5b1940a26f6b6e92cc2f11a0e1deef31 2024-11-17T15:37:49,419 INFO [StoreOpener-5b1940a26f6b6e92cc2f11a0e1deef31-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5b1940a26f6b6e92cc2f11a0e1deef31 columnFamilyName B 2024-11-17T15:37:49,419 DEBUG [StoreOpener-5b1940a26f6b6e92cc2f11a0e1deef31-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:49,419 INFO [StoreOpener-5b1940a26f6b6e92cc2f11a0e1deef31-1 {}] regionserver.HStore(327): Store=5b1940a26f6b6e92cc2f11a0e1deef31/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T15:37:49,419 INFO [StoreOpener-5b1940a26f6b6e92cc2f11a0e1deef31-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 5b1940a26f6b6e92cc2f11a0e1deef31 2024-11-17T15:37:49,420 INFO [StoreOpener-5b1940a26f6b6e92cc2f11a0e1deef31-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5b1940a26f6b6e92cc2f11a0e1deef31 columnFamilyName C 2024-11-17T15:37:49,420 DEBUG [StoreOpener-5b1940a26f6b6e92cc2f11a0e1deef31-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:37:49,421 INFO [StoreOpener-5b1940a26f6b6e92cc2f11a0e1deef31-1 {}] regionserver.HStore(327): Store=5b1940a26f6b6e92cc2f11a0e1deef31/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T15:37:49,421 DEBUG [RS_OPEN_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(1038): replaying wal for 5b1940a26f6b6e92cc2f11a0e1deef31 2024-11-17T15:37:49,421 DEBUG [RS_OPEN_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31 2024-11-17T15:37:49,422 DEBUG [RS_OPEN_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31 2024-11-17T15:37:49,422 DEBUG [RS_OPEN_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(1048): stopping wal replay for 5b1940a26f6b6e92cc2f11a0e1deef31 2024-11-17T15:37:49,422 DEBUG [RS_OPEN_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(1060): Cleaning up temporary data for 5b1940a26f6b6e92cc2f11a0e1deef31 2024-11-17T15:37:49,423 DEBUG [RS_OPEN_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-17T15:37:49,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=65 2024-11-17T15:37:49,424 DEBUG [RS_OPEN_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(1093): writing seq id for 5b1940a26f6b6e92cc2f11a0e1deef31 2024-11-17T15:37:49,426 DEBUG [RS_OPEN_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-17T15:37:49,426 INFO [RS_OPEN_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(1114): Opened 5b1940a26f6b6e92cc2f11a0e1deef31; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74035333, jitterRate=0.10321243107318878}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-17T15:37:49,426 DEBUG [RS_OPEN_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5b1940a26f6b6e92cc2f11a0e1deef31 2024-11-17T15:37:49,427 DEBUG [RS_OPEN_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(1006): Region open journal for 5b1940a26f6b6e92cc2f11a0e1deef31: Running coprocessor pre-open hook at 1731857869415Writing region info on filesystem at 1731857869415Initializing all the Stores at 1731857869416 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731857869416Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731857869416Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731857869416Cleaning up temporary data from old regions at 1731857869422 (+6 ms)Running coprocessor post-open hooks at 1731857869426 (+4 ms)Region opened successfully at 1731857869427 (+1 ms) 2024-11-17T15:37:49,428 INFO [RS_OPEN_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., pid=67, masterSystemTime=1731857869412 2024-11-17T15:37:49,429 DEBUG [RS_OPEN_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31. 2024-11-17T15:37:49,429 INFO [RS_OPEN_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31. 2024-11-17T15:37:49,430 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=66 updating hbase:meta row=5b1940a26f6b6e92cc2f11a0e1deef31, regionState=OPEN, openSeqNum=2, regionLocation=7a780d55532c,32955,1731857800648 2024-11-17T15:37:49,431 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=67, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 because future has completed 2024-11-17T15:37:49,433 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-11-17T15:37:49,433 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; OpenRegionProcedure 5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 in 172 msec 2024-11-17T15:37:49,434 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=66, resume processing ppid=65 2024-11-17T15:37:49,435 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, ppid=65, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5b1940a26f6b6e92cc2f11a0e1deef31, ASSIGN in 327 msec 2024-11-17T15:37:49,435 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=65, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-17T15:37:49,435 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731857869435"}]},"ts":"1731857869435"} 2024-11-17T15:37:49,436 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-17T15:37:49,437 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=65, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-17T15:37:49,438 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1520 sec 2024-11-17T15:37:50,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=65 2024-11-17T15:37:50,433 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-17T15:37:50,434 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61a754db, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:37:50,434 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7a780d55532c,44223,-1 for getting cluster id 2024-11-17T15:37:50,434 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T15:37:50,435 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e01f4cd2-8971-4a73-861a-2ca0deb4f76e' 2024-11-17T15:37:50,436 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T15:37:50,436 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e01f4cd2-8971-4a73-861a-2ca0deb4f76e" 2024-11-17T15:37:50,436 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f9248b6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:37:50,436 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7a780d55532c,44223,-1] 2024-11-17T15:37:50,436 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T15:37:50,436 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:37:50,437 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57914, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T15:37:50,438 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73a73d3b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:37:50,438 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T15:37:50,439 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7a780d55532c,32955,1731857800648, seqNum=-1] 2024-11-17T15:37:50,439 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T15:37:50,440 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35874, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T15:37:50,441 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-17T15:37:50,441 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is 7a780d55532c,44223,1731857799978 2024-11-17T15:37:50,441 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@55a57c12 2024-11-17T15:37:50,442 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-17T15:37:50,442 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57930, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-17T15:37:50,443 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50502613, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:37:50,443 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7a780d55532c,44223,-1 for getting cluster id 2024-11-17T15:37:50,443 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T15:37:50,444 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e01f4cd2-8971-4a73-861a-2ca0deb4f76e' 2024-11-17T15:37:50,444 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T15:37:50,444 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e01f4cd2-8971-4a73-861a-2ca0deb4f76e" 2024-11-17T15:37:50,444 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b5024a6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:37:50,444 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7a780d55532c,44223,-1] 2024-11-17T15:37:50,444 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T15:37:50,445 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:37:50,445 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57958, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T15:37:50,446 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59f50964, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:37:50,446 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@554689fb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:37:50,446 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7a780d55532c,44223,-1 for getting cluster id 2024-11-17T15:37:50,446 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T15:37:50,447 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e01f4cd2-8971-4a73-861a-2ca0deb4f76e' 2024-11-17T15:37:50,447 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T15:37:50,447 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e01f4cd2-8971-4a73-861a-2ca0deb4f76e" 2024-11-17T15:37:50,447 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11fb097b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:37:50,447 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7a780d55532c,44223,-1] 2024-11-17T15:37:50,448 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T15:37:50,448 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:37:50,448 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57976, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T15:37:50,448 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d106e1d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:37:50,449 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72325ab8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:37:50,449 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7a780d55532c,44223,-1 for getting cluster id 2024-11-17T15:37:50,449 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T15:37:50,450 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e01f4cd2-8971-4a73-861a-2ca0deb4f76e' 2024-11-17T15:37:50,450 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T15:37:50,450 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e01f4cd2-8971-4a73-861a-2ca0deb4f76e" 2024-11-17T15:37:50,450 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@295ce45c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:37:50,450 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7a780d55532c,44223,-1] 2024-11-17T15:37:50,450 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T15:37:50,450 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:37:50,451 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57986, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T15:37:50,451 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5345781, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:37:50,452 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6686bc48, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:37:50,452 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7a780d55532c,44223,-1 for getting cluster id 2024-11-17T15:37:50,452 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T15:37:50,452 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e01f4cd2-8971-4a73-861a-2ca0deb4f76e' 2024-11-17T15:37:50,452 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T15:37:50,453 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e01f4cd2-8971-4a73-861a-2ca0deb4f76e" 2024-11-17T15:37:50,453 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28a27cbe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:37:50,453 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7a780d55532c,44223,-1] 2024-11-17T15:37:50,453 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T15:37:50,453 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:37:50,453 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57998, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T15:37:50,454 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9ae7b1e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:37:50,454 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d70004e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:37:50,454 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7a780d55532c,44223,-1 for getting cluster id 2024-11-17T15:37:50,455 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T15:37:50,455 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e01f4cd2-8971-4a73-861a-2ca0deb4f76e' 2024-11-17T15:37:50,455 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T15:37:50,455 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e01f4cd2-8971-4a73-861a-2ca0deb4f76e" 2024-11-17T15:37:50,455 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@60d41bfc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:37:50,455 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7a780d55532c,44223,-1] 2024-11-17T15:37:50,456 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T15:37:50,456 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:37:50,456 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58012, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T15:37:50,457 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37a04fe3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:37:50,457 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@675e61a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:37:50,457 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7a780d55532c,44223,-1 for getting cluster id 2024-11-17T15:37:50,457 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T15:37:50,458 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e01f4cd2-8971-4a73-861a-2ca0deb4f76e' 2024-11-17T15:37:50,458 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T15:37:50,458 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e01f4cd2-8971-4a73-861a-2ca0deb4f76e" 2024-11-17T15:37:50,458 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72a17320, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:37:50,458 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7a780d55532c,44223,-1] 2024-11-17T15:37:50,458 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T15:37:50,458 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:37:50,459 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58024, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T15:37:50,459 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a5b5446, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:37:50,460 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b9635b3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:37:50,460 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7a780d55532c,44223,-1 for getting cluster id 2024-11-17T15:37:50,460 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T15:37:50,461 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e01f4cd2-8971-4a73-861a-2ca0deb4f76e' 2024-11-17T15:37:50,461 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T15:37:50,461 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e01f4cd2-8971-4a73-861a-2ca0deb4f76e" 2024-11-17T15:37:50,461 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@80e25a5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:37:50,461 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7a780d55532c,44223,-1] 2024-11-17T15:37:50,461 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T15:37:50,461 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:37:50,462 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58038, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T15:37:50,462 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@206a7f1a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:37:50,463 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2611025, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:37:50,463 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7a780d55532c,44223,-1 for getting cluster id 2024-11-17T15:37:50,463 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T15:37:50,463 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e01f4cd2-8971-4a73-861a-2ca0deb4f76e' 2024-11-17T15:37:50,463 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T15:37:50,463 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e01f4cd2-8971-4a73-861a-2ca0deb4f76e" 2024-11-17T15:37:50,464 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@593eea78, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:37:50,464 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7a780d55532c,44223,-1] 2024-11-17T15:37:50,464 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T15:37:50,464 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:37:50,464 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58068, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T15:37:50,465 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ebeec1e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:37:50,465 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8a8ae50, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:37:50,465 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7a780d55532c,44223,-1 for getting cluster id 2024-11-17T15:37:50,466 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T15:37:50,466 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e01f4cd2-8971-4a73-861a-2ca0deb4f76e' 2024-11-17T15:37:50,466 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T15:37:50,466 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e01f4cd2-8971-4a73-861a-2ca0deb4f76e" 2024-11-17T15:37:50,466 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35ffd00d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:37:50,466 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7a780d55532c,44223,-1] 2024-11-17T15:37:50,466 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T15:37:50,467 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:37:50,467 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58084, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T15:37:50,467 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3780bf57, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:37:50,468 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@471771d0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:37:50,468 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7a780d55532c,44223,-1 for getting cluster id 2024-11-17T15:37:50,468 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T15:37:50,468 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e01f4cd2-8971-4a73-861a-2ca0deb4f76e' 2024-11-17T15:37:50,469 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T15:37:50,469 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e01f4cd2-8971-4a73-861a-2ca0deb4f76e" 2024-11-17T15:37:50,469 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@392c3ddf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:37:50,469 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7a780d55532c,44223,-1] 2024-11-17T15:37:50,469 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T15:37:50,469 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:37:50,470 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58116, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T15:37:50,470 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27ede31c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:37:50,471 DEBUG [Thread-1150 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T15:37:50,471 DEBUG [Thread-1154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T15:37:50,471 DEBUG [Thread-1165 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T15:37:50,471 DEBUG [Thread-1159 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T15:37:50,471 DEBUG [Thread-1163 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T15:37:50,471 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T15:37:50,472 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7a780d55532c,32955,1731857800648, seqNum=-1] 2024-11-17T15:37:50,472 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7a780d55532c,32955,1731857800648, seqNum=-1] 2024-11-17T15:37:50,472 DEBUG [Thread-1157 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T15:37:50,472 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7a780d55532c,32955,1731857800648, seqNum=-1] 2024-11-17T15:37:50,472 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T15:37:50,472 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T15:37:50,472 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T15:37:50,472 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7a780d55532c,32955,1731857800648, seqNum=-1] 2024-11-17T15:37:50,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-11-17T15:37:50,472 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T15:37:50,473 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35884, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T15:37:50,473 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35886, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T15:37:50,473 DEBUG [Thread-1146 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T15:37:50,473 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35902, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T15:37:50,473 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35908, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T15:37:50,474 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7a780d55532c,32955,1731857800648, seqNum=-1] 2024-11-17T15:37:50,474 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T15:37:50,474 DEBUG [Thread-1161 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T15:37:50,474 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T15:37:50,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-17T15:37:50,474 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7a780d55532c,32955,1731857800648, seqNum=-1] 2024-11-17T15:37:50,475 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T15:37:50,475 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7a780d55532c,32955,1731857800648, seqNum=-1] 2024-11-17T15:37:50,475 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2] 2024-11-17T15:37:50,475 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T15:37:50,475 DEBUG [Thread-1146 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7a780d55532c,32955,1731857800648, seqNum=-1] 2024-11-17T15:37:50,475 DEBUG [Thread-1146 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T15:37:50,475 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35922, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T15:37:50,476 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35924, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T15:37:50,476 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T15:37:50,476 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T15:37:50,476 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35944, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T15:37:50,476 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2] 2024-11-17T15:37:50,476 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35940, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T15:37:50,477 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2] 2024-11-17T15:37:50,477 DEBUG [Thread-1148 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T15:37:50,478 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7a780d55532c,32955,1731857800648, seqNum=-1] 2024-11-17T15:37:50,478 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T15:37:50,479 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2] 2024-11-17T15:37:50,479 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2] 2024-11-17T15:37:50,480 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2] 2024-11-17T15:37:50,480 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35954, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T15:37:50,480 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2] 2024-11-17T15:37:50,480 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2] 2024-11-17T15:37:50,484 DEBUG [Thread-1152 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T15:37:50,485 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7a780d55532c,32955,1731857800648, seqNum=-1] 2024-11-17T15:37:50,485 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T15:37:50,486 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35958, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T15:37:50,490 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2] 2024-11-17T15:37:50,490 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2] 2024-11-17T15:37:50,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(8855): Flush requested on 5b1940a26f6b6e92cc2f11a0e1deef31 2024-11-17T15:37:50,539 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T15:37:50,540 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5b1940a26f6b6e92cc2f11a0e1deef31 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-17T15:37:50,569 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/A/9268311b18344df1966a86d170925706 is 50, key is test_row_0/A:col10/1731857870538/Put/seqid=0 2024-11-17T15:37:50,578 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:50,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35940 deadline: 1731857930573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:50,579 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:50,579 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:50,579 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:37:50,581 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:50,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35958 deadline: 1731857930574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:50,582 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:50,582 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:50,582 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:50,582 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:37:50,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35884 deadline: 1731857930576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:50,582 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:50,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35902 deadline: 1731857930579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:50,583 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:50,583 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:50,583 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:37:50,583 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:50,583 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:50,583 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:37:50,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-17T15:37:50,583 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:50,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35954 deadline: 1731857930579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:50,584 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:50,584 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:50,584 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:37:50,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742072_1248 (size=12001) 2024-11-17T15:37:50,599 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/A/9268311b18344df1966a86d170925706 2024-11-17T15:37:50,633 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-17T15:37:50,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31. 2024-11-17T15:37:50,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31. as already flushing 2024-11-17T15:37:50,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31. 2024-11-17T15:37:50,633 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 5b1940a26f6b6e92cc2f11a0e1deef31, NAME => 'TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:37:50,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 5b1940a26f6b6e92cc2f11a0e1deef31, NAME => 'TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:37:50,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b1940a26f6b6e92cc2f11a0e1deef31, NAME => 'TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b1940a26f6b6e92cc2f11a0e1deef31, NAME => 'TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:37:50,649 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/B/ed7fc068ae084003bff17231c9a342bb is 50, key is test_row_0/B:col10/1731857870538/Put/seqid=0 2024-11-17T15:37:50,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742073_1249 (size=12001) 2024-11-17T15:37:50,685 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:50,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35940 deadline: 1731857930683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:50,686 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:50,686 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:50,686 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:37:50,695 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:50,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35958 deadline: 1731857930693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:50,695 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:50,695 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:50,695 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:50,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35884 deadline: 1731857930693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:50,695 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:37:50,696 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:50,696 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:50,696 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:37:50,697 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:50,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35954 deadline: 1731857930694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:50,697 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:50,697 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:50,697 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:37:50,696 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:50,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35902 deadline: 1731857930694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:50,698 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:50,698 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:50,698 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:37:50,787 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-17T15:37:50,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31. 2024-11-17T15:37:50,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31. as already flushing 2024-11-17T15:37:50,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31. 2024-11-17T15:37:50,787 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 5b1940a26f6b6e92cc2f11a0e1deef31, NAME => 'TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:37:50,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 5b1940a26f6b6e92cc2f11a0e1deef31, NAME => 'TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:37:50,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b1940a26f6b6e92cc2f11a0e1deef31, NAME => 'TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b1940a26f6b6e92cc2f11a0e1deef31, NAME => 'TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:37:50,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-17T15:37:50,895 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:50,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35940 deadline: 1731857930893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:50,895 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:50,895 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:50,895 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:37:50,906 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:50,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35958 deadline: 1731857930903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:50,906 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:50,906 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:50,907 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:50,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35884 deadline: 1731857930904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:50,907 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:37:50,907 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:50,907 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:50,907 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:50,907 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:37:50,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35954 deadline: 1731857930904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:50,907 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:50,907 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:50,907 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:50,907 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:37:50,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35902 deadline: 1731857930904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:50,907 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:50,908 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:50,908 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:37:50,940 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-17T15:37:50,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31. 2024-11-17T15:37:50,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31. as already flushing 2024-11-17T15:37:50,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31. 2024-11-17T15:37:50,940 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 5b1940a26f6b6e92cc2f11a0e1deef31, NAME => 'TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:37:50,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 5b1940a26f6b6e92cc2f11a0e1deef31, NAME => 'TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:37:50,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b1940a26f6b6e92cc2f11a0e1deef31, NAME => 'TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b1940a26f6b6e92cc2f11a0e1deef31, NAME => 'TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:37:51,062 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/B/ed7fc068ae084003bff17231c9a342bb 2024-11-17T15:37:51,091 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/C/fc36d5cb8bdc46d798f897b0aeb41488 is 50, key is test_row_0/C:col10/1731857870538/Put/seqid=0 2024-11-17T15:37:51,093 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-17T15:37:51,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31. 2024-11-17T15:37:51,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31. as already flushing 2024-11-17T15:37:51,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31. 2024-11-17T15:37:51,094 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 5b1940a26f6b6e92cc2f11a0e1deef31, NAME => 'TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:37:51,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 5b1940a26f6b6e92cc2f11a0e1deef31, NAME => 'TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:37:51,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b1940a26f6b6e92cc2f11a0e1deef31, NAME => 'TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b1940a26f6b6e92cc2f11a0e1deef31, NAME => 'TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:37:51,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742074_1250 (size=12001) 2024-11-17T15:37:51,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-17T15:37:51,205 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:51,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35940 deadline: 1731857931203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:51,206 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:51,206 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:51,206 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:37:51,215 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:51,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35958 deadline: 1731857931213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:51,215 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:51,215 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:51,215 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:51,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35884 deadline: 1731857931213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:51,215 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:37:51,215 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:51,216 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:51,216 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:37:51,216 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:51,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35954 deadline: 1731857931213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:51,216 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:51,216 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:51,216 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:51,216 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:37:51,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35902 deadline: 1731857931214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:51,216 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:51,216 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:51,216 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:37:51,246 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-17T15:37:51,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31. 2024-11-17T15:37:51,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31. as already flushing 2024-11-17T15:37:51,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31. 2024-11-17T15:37:51,247 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 5b1940a26f6b6e92cc2f11a0e1deef31, NAME => 'TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:37:51,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 5b1940a26f6b6e92cc2f11a0e1deef31, NAME => 'TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:37:51,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b1940a26f6b6e92cc2f11a0e1deef31, NAME => 'TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b1940a26f6b6e92cc2f11a0e1deef31, NAME => 'TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:37:51,399 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-17T15:37:51,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31. 2024-11-17T15:37:51,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31. as already flushing 2024-11-17T15:37:51,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31. 2024-11-17T15:37:51,400 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 5b1940a26f6b6e92cc2f11a0e1deef31, NAME => 'TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:37:51,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 5b1940a26f6b6e92cc2f11a0e1deef31, NAME => 'TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:37:51,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b1940a26f6b6e92cc2f11a0e1deef31, NAME => 'TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b1940a26f6b6e92cc2f11a0e1deef31, NAME => 'TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:37:51,498 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/C/fc36d5cb8bdc46d798f897b0aeb41488 2024-11-17T15:37:51,503 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/A/9268311b18344df1966a86d170925706 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/9268311b18344df1966a86d170925706 2024-11-17T15:37:51,508 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/9268311b18344df1966a86d170925706, entries=150, sequenceid=14, filesize=11.7 K 2024-11-17T15:37:51,508 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/B/ed7fc068ae084003bff17231c9a342bb as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/ed7fc068ae084003bff17231c9a342bb 2024-11-17T15:37:51,513 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/ed7fc068ae084003bff17231c9a342bb, entries=150, sequenceid=14, filesize=11.7 K 2024-11-17T15:37:51,514 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/C/fc36d5cb8bdc46d798f897b0aeb41488 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/fc36d5cb8bdc46d798f897b0aeb41488 2024-11-17T15:37:51,518 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/fc36d5cb8bdc46d798f897b0aeb41488, entries=150, sequenceid=14, filesize=11.7 K 2024-11-17T15:37:51,518 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=140.89 KB/144270 for 5b1940a26f6b6e92cc2f11a0e1deef31 in 979ms, sequenceid=14, compaction requested=false 2024-11-17T15:37:51,519 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-17T15:37:51,519 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5b1940a26f6b6e92cc2f11a0e1deef31: 2024-11-17T15:37:51,553 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-17T15:37:51,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31. 2024-11-17T15:37:51,553 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing 5b1940a26f6b6e92cc2f11a0e1deef31 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-17T15:37:51,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/A/8579fd292b254d56a1198dfb452ccb7d is 50, key is test_row_0/A:col10/1731857870577/Put/seqid=0 2024-11-17T15:37:51,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742075_1251 (size=12001) 2024-11-17T15:37:51,589 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/A/8579fd292b254d56a1198dfb452ccb7d 2024-11-17T15:37:51,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-17T15:37:51,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/B/6e6f555724274f3fb09c15f9774e92fd is 50, key is test_row_0/B:col10/1731857870577/Put/seqid=0 2024-11-17T15:37:51,642 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742076_1252 (size=12001) 2024-11-17T15:37:51,643 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/B/6e6f555724274f3fb09c15f9774e92fd 2024-11-17T15:37:51,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/C/59ade488e18b414ebca0bb91a36e4b43 is 50, key is test_row_0/C:col10/1731857870577/Put/seqid=0 2024-11-17T15:37:51,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742077_1253 (size=12001) 2024-11-17T15:37:51,697 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/C/59ade488e18b414ebca0bb91a36e4b43 2024-11-17T15:37:51,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/A/8579fd292b254d56a1198dfb452ccb7d as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/8579fd292b254d56a1198dfb452ccb7d 2024-11-17T15:37:51,711 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/8579fd292b254d56a1198dfb452ccb7d, entries=150, sequenceid=38, filesize=11.7 K 2024-11-17T15:37:51,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/B/6e6f555724274f3fb09c15f9774e92fd as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/6e6f555724274f3fb09c15f9774e92fd 2024-11-17T15:37:51,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(8855): Flush requested on 5b1940a26f6b6e92cc2f11a0e1deef31 2024-11-17T15:37:51,716 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31. as already flushing 2024-11-17T15:37:51,721 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/6e6f555724274f3fb09c15f9774e92fd, entries=150, sequenceid=38, filesize=11.7 K 2024-11-17T15:37:51,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/C/59ade488e18b414ebca0bb91a36e4b43 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/59ade488e18b414ebca0bb91a36e4b43 2024-11-17T15:37:51,729 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/59ade488e18b414ebca0bb91a36e4b43, entries=150, sequenceid=38, filesize=11.7 K 2024-11-17T15:37:51,730 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=40.25 KB/41220 for 5b1940a26f6b6e92cc2f11a0e1deef31 in 177ms, sequenceid=38, compaction requested=false 2024-11-17T15:37:51,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for 5b1940a26f6b6e92cc2f11a0e1deef31: 2024-11-17T15:37:51,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31. 2024-11-17T15:37:51,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-11-17T15:37:51,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-11-17T15:37:51,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(8855): Flush requested on 5b1940a26f6b6e92cc2f11a0e1deef31 2024-11-17T15:37:51,732 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T15:37:51,732 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5b1940a26f6b6e92cc2f11a0e1deef31 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-17T15:37:51,734 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-11-17T15:37:51,734 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2560 sec 2024-11-17T15:37:51,737 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 1.2640 sec 2024-11-17T15:37:51,742 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/A/e175d4579b5e40a5923b52311b44665a is 50, key is test_row_0/A:col10/1731857871728/Put/seqid=0 2024-11-17T15:37:51,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742078_1254 (size=12001) 2024-11-17T15:37:51,771 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:51,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35954 deadline: 1731857931766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:51,772 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:51,772 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:51,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35958 deadline: 1731857931766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:51,772 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:51,772 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:37:51,772 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:51,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35940 deadline: 1731857931766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:51,773 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:51,773 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:51,773 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:37:51,773 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:51,773 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:51,773 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:51,773 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:37:51,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35884 deadline: 1731857931767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:51,773 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:51,773 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:51,773 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:37:51,776 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:51,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35902 deadline: 1731857931771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:51,776 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:51,776 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:51,776 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:37:51,886 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:51,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35954 deadline: 1731857931883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:51,886 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:51,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35958 deadline: 1731857931883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:51,886 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:51,886 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:51,886 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:37:51,886 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:51,887 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:51,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35940 deadline: 1731857931883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:51,887 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:51,887 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:37:51,887 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:51,887 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:51,887 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:51,887 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:37:51,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35902 deadline: 1731857931883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:51,887 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:51,887 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:51,887 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:51,887 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:37:51,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35884 deadline: 1731857931883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:51,888 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:51,888 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:51,888 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:37:52,094 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:52,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35954 deadline: 1731857932093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:52,094 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:52,094 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:52,094 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:37:52,096 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:52,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35958 deadline: 1731857932093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:52,097 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:52,097 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:52,097 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:37:52,097 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:52,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35940 deadline: 1731857932093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:52,097 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:52,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35902 deadline: 1731857932093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:52,097 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:52,097 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:52,097 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:52,097 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:37:52,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35884 deadline: 1731857932094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:52,098 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:52,098 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:52,098 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:37:52,098 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:52,098 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:52,098 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:37:52,163 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/A/e175d4579b5e40a5923b52311b44665a 2024-11-17T15:37:52,170 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/B/dbfad03ca1ba4b49bf880460062387ca is 50, key is test_row_0/B:col10/1731857871728/Put/seqid=0 2024-11-17T15:37:52,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742079_1255 (size=12001) 2024-11-17T15:37:52,404 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:52,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35954 deadline: 1731857932403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:52,405 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:52,405 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:52,405 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:52,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35958 deadline: 1731857932403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:52,405 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:37:52,405 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:52,405 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:52,405 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:52,405 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:37:52,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35940 deadline: 1731857932403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:52,405 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:52,405 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:52,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35884 deadline: 1731857932403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:52,405 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:52,405 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:37:52,405 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:52,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35902 deadline: 1731857932403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:52,406 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:52,406 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:52,406 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:37:52,406 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:52,406 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:52,406 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:37:52,574 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/B/dbfad03ca1ba4b49bf880460062387ca 2024-11-17T15:37:52,583 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/C/fb883a228d4e4bc3be7a2fe3e45b5ed6 is 50, key is test_row_0/C:col10/1731857871728/Put/seqid=0 2024-11-17T15:37:52,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742080_1256 (size=12001) 2024-11-17T15:37:52,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-17T15:37:52,623 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T15:37:52,624 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T15:37:52,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-11-17T15:37:52,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-17T15:37:52,626 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T15:37:52,627 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T15:37:52,627 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T15:37:52,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-17T15:37:52,779 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-17T15:37:52,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31. 2024-11-17T15:37:52,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31. as already flushing 2024-11-17T15:37:52,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31. 2024-11-17T15:37:52,780 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 5b1940a26f6b6e92cc2f11a0e1deef31, NAME => 'TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:37:52,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 5b1940a26f6b6e92cc2f11a0e1deef31, NAME => 'TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:37:52,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b1940a26f6b6e92cc2f11a0e1deef31, NAME => 'TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b1940a26f6b6e92cc2f11a0e1deef31, NAME => 'TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:37:52,915 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:52,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35954 deadline: 1731857932913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:52,916 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:52,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35958 deadline: 1731857932913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:52,916 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:52,916 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:52,916 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:37:52,916 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:52,916 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:52,916 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:52,916 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:37:52,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35940 deadline: 1731857932913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:52,916 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:52,916 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:52,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35884 deadline: 1731857932913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:52,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35902 deadline: 1731857932913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:52,917 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:52,917 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:52,917 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:37:52,917 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:52,917 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:52,917 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:37:52,917 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:52,917 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:52,917 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:37:52,933 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-17T15:37:52,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31. 2024-11-17T15:37:52,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31. as already flushing 2024-11-17T15:37:52,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31. 2024-11-17T15:37:52,933 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 5b1940a26f6b6e92cc2f11a0e1deef31, NAME => 'TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:37:52,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 5b1940a26f6b6e92cc2f11a0e1deef31, NAME => 'TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:37:52,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b1940a26f6b6e92cc2f11a0e1deef31, NAME => 'TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b1940a26f6b6e92cc2f11a0e1deef31, NAME => 'TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:37:52,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-17T15:37:52,994 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/C/fb883a228d4e4bc3be7a2fe3e45b5ed6 2024-11-17T15:37:53,003 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/A/e175d4579b5e40a5923b52311b44665a as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/e175d4579b5e40a5923b52311b44665a 2024-11-17T15:37:53,017 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/e175d4579b5e40a5923b52311b44665a, entries=150, sequenceid=50, filesize=11.7 K 2024-11-17T15:37:53,019 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/B/dbfad03ca1ba4b49bf880460062387ca as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/dbfad03ca1ba4b49bf880460062387ca 2024-11-17T15:37:53,024 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/dbfad03ca1ba4b49bf880460062387ca, entries=150, sequenceid=50, filesize=11.7 K 2024-11-17T15:37:53,025 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreScanner(997): StoreScanner already has the close lock. There is no need to updateReaders 2024-11-17T15:37:53,027 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/C/fb883a228d4e4bc3be7a2fe3e45b5ed6 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/fb883a228d4e4bc3be7a2fe3e45b5ed6 2024-11-17T15:37:53,034 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/fb883a228d4e4bc3be7a2fe3e45b5ed6, entries=150, sequenceid=50, filesize=11.7 K 2024-11-17T15:37:53,040 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 5b1940a26f6b6e92cc2f11a0e1deef31 in 1308ms, sequenceid=50, compaction requested=true 2024-11-17T15:37:53,040 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5b1940a26f6b6e92cc2f11a0e1deef31: 2024-11-17T15:37:53,040 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T15:37:53,041 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5b1940a26f6b6e92cc2f11a0e1deef31:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T15:37:53,041 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T15:37:53,041 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5b1940a26f6b6e92cc2f11a0e1deef31:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T15:37:53,041 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T15:37:53,041 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5b1940a26f6b6e92cc2f11a0e1deef31:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T15:37:53,041 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T15:37:53,041 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T15:37:53,042 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T15:37:53,042 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T15:37:53,042 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1541): 5b1940a26f6b6e92cc2f11a0e1deef31/A is initiating minor compaction (all files) 2024-11-17T15:37:53,042 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1541): 5b1940a26f6b6e92cc2f11a0e1deef31/B is initiating minor compaction (all files) 2024-11-17T15:37:53,042 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5b1940a26f6b6e92cc2f11a0e1deef31/B in TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31. 2024-11-17T15:37:53,043 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5b1940a26f6b6e92cc2f11a0e1deef31/A in TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31. 2024-11-17T15:37:53,043 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/ed7fc068ae084003bff17231c9a342bb, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/6e6f555724274f3fb09c15f9774e92fd, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/dbfad03ca1ba4b49bf880460062387ca] into tmpdir=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp, totalSize=35.2 K 2024-11-17T15:37:53,043 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/9268311b18344df1966a86d170925706, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/8579fd292b254d56a1198dfb452ccb7d, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/e175d4579b5e40a5923b52311b44665a] into tmpdir=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp, totalSize=35.2 K 2024-11-17T15:37:53,043 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9268311b18344df1966a86d170925706, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731857870484 2024-11-17T15:37:53,043 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting ed7fc068ae084003bff17231c9a342bb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731857870484 2024-11-17T15:37:53,044 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8579fd292b254d56a1198dfb452ccb7d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1731857870572 2024-11-17T15:37:53,044 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting 6e6f555724274f3fb09c15f9774e92fd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1731857870572 2024-11-17T15:37:53,044 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting e175d4579b5e40a5923b52311b44665a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731857871728 2024-11-17T15:37:53,045 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting dbfad03ca1ba4b49bf880460062387ca, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731857871728 2024-11-17T15:37:53,056 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5b1940a26f6b6e92cc2f11a0e1deef31#A#compaction#206 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T15:37:53,057 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/A/934518f8976a4194ad4f338b0602ffd0 is 50, key is test_row_0/A:col10/1731857871728/Put/seqid=0 2024-11-17T15:37:53,066 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5b1940a26f6b6e92cc2f11a0e1deef31#B#compaction#207 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T15:37:53,067 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/B/8b53743e04b74406b5e549c1aeeb997c is 50, key is test_row_0/B:col10/1731857871728/Put/seqid=0 2024-11-17T15:37:53,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742081_1257 (size=12104) 2024-11-17T15:37:53,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742082_1258 (size=12104) 2024-11-17T15:37:53,081 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/A/934518f8976a4194ad4f338b0602ffd0 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/934518f8976a4194ad4f338b0602ffd0 2024-11-17T15:37:53,088 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/B/8b53743e04b74406b5e549c1aeeb997c as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/8b53743e04b74406b5e549c1aeeb997c 2024-11-17T15:37:53,089 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5b1940a26f6b6e92cc2f11a0e1deef31/A of 5b1940a26f6b6e92cc2f11a0e1deef31 into 934518f8976a4194ad4f338b0602ffd0(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T15:37:53,089 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5b1940a26f6b6e92cc2f11a0e1deef31: 2024-11-17T15:37:53,089 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., storeName=5b1940a26f6b6e92cc2f11a0e1deef31/A, priority=13, startTime=1731857873040; duration=0sec 2024-11-17T15:37:53,089 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T15:37:53,089 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5b1940a26f6b6e92cc2f11a0e1deef31:A 2024-11-17T15:37:53,089 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T15:37:53,090 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-17T15:37:53,090 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T15:37:53,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31. 2024-11-17T15:37:53,091 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1541): 5b1940a26f6b6e92cc2f11a0e1deef31/C is initiating minor compaction (all files) 2024-11-17T15:37:53,091 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5b1940a26f6b6e92cc2f11a0e1deef31/C in TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31. 2024-11-17T15:37:53,091 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing 5b1940a26f6b6e92cc2f11a0e1deef31 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-17T15:37:53,091 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/fc36d5cb8bdc46d798f897b0aeb41488, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/59ade488e18b414ebca0bb91a36e4b43, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/fb883a228d4e4bc3be7a2fe3e45b5ed6] into tmpdir=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp, totalSize=35.2 K 2024-11-17T15:37:53,091 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting fc36d5cb8bdc46d798f897b0aeb41488, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731857870484 2024-11-17T15:37:53,091 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 59ade488e18b414ebca0bb91a36e4b43, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1731857870572 2024-11-17T15:37:53,093 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting fb883a228d4e4bc3be7a2fe3e45b5ed6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731857871728 2024-11-17T15:37:53,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/A/000b01d9ad3c462e943c54bd46627c66 is 50, key is test_row_0/A:col10/1731857871766/Put/seqid=0 2024-11-17T15:37:53,100 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5b1940a26f6b6e92cc2f11a0e1deef31/B of 5b1940a26f6b6e92cc2f11a0e1deef31 into 8b53743e04b74406b5e549c1aeeb997c(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T15:37:53,100 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5b1940a26f6b6e92cc2f11a0e1deef31: 2024-11-17T15:37:53,100 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., storeName=5b1940a26f6b6e92cc2f11a0e1deef31/B, priority=13, startTime=1731857873041; duration=0sec 2024-11-17T15:37:53,101 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T15:37:53,101 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5b1940a26f6b6e92cc2f11a0e1deef31:B 2024-11-17T15:37:53,104 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742083_1259 (size=12001) 2024-11-17T15:37:53,107 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5b1940a26f6b6e92cc2f11a0e1deef31#C#compaction#209 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T15:37:53,107 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/C/8078048adee444c68dda5ee03376915b is 50, key is test_row_0/C:col10/1731857871728/Put/seqid=0 2024-11-17T15:37:53,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742084_1260 (size=12104) 2024-11-17T15:37:53,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-17T15:37:53,449 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-17T15:37:53,505 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/A/000b01d9ad3c462e943c54bd46627c66 2024-11-17T15:37:53,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/B/c246c40addd44edfa90e375ff06ad5cb is 50, key is test_row_0/B:col10/1731857871766/Put/seqid=0 2024-11-17T15:37:53,520 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/C/8078048adee444c68dda5ee03376915b as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/8078048adee444c68dda5ee03376915b 2024-11-17T15:37:53,527 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5b1940a26f6b6e92cc2f11a0e1deef31/C of 5b1940a26f6b6e92cc2f11a0e1deef31 into 8078048adee444c68dda5ee03376915b(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T15:37:53,527 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5b1940a26f6b6e92cc2f11a0e1deef31: 2024-11-17T15:37:53,527 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., storeName=5b1940a26f6b6e92cc2f11a0e1deef31/C, priority=13, startTime=1731857873041; duration=0sec 2024-11-17T15:37:53,527 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T15:37:53,527 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5b1940a26f6b6e92cc2f11a0e1deef31:C 2024-11-17T15:37:53,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742085_1261 (size=12001) 2024-11-17T15:37:53,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-17T15:37:53,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(8855): Flush requested on 5b1940a26f6b6e92cc2f11a0e1deef31 2024-11-17T15:37:53,926 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31. as already flushing 2024-11-17T15:37:53,935 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/B/c246c40addd44edfa90e375ff06ad5cb 2024-11-17T15:37:53,938 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:53,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35940 deadline: 1731857933933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:53,939 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:53,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35902 deadline: 1731857933933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:53,939 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:53,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35958 deadline: 1731857933936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:53,940 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:53,940 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:53,940 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:37:53,939 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 7a780d55532c,32955,1731857800648 for 'test_row_2' in 5b1940a26f6b6e92cc2f11a0e1deef31 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2168 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T15:37:53,940 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:53,940 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:53,940 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:37:53,940 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:53,940 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:53,940 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:37:53,943 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:53,944 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:53,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35954 deadline: 1731857933938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:53,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35884 deadline: 1731857933937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:53,944 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:53,944 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:53,944 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:53,944 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:37:53,944 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:53,944 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:37:53,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/C/cd1cb539dca04df3b5a2ffa6341cafff is 50, key is test_row_0/C:col10/1731857871766/Put/seqid=0 2024-11-17T15:37:53,948 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742086_1262 (size=12001) 2024-11-17T15:37:53,948 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/C/cd1cb539dca04df3b5a2ffa6341cafff 2024-11-17T15:37:53,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/A/000b01d9ad3c462e943c54bd46627c66 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/000b01d9ad3c462e943c54bd46627c66 2024-11-17T15:37:53,959 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/000b01d9ad3c462e943c54bd46627c66, entries=150, sequenceid=75, filesize=11.7 K 2024-11-17T15:37:53,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/B/c246c40addd44edfa90e375ff06ad5cb as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/c246c40addd44edfa90e375ff06ad5cb 2024-11-17T15:37:53,965 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/c246c40addd44edfa90e375ff06ad5cb, entries=150, sequenceid=75, filesize=11.7 K 2024-11-17T15:37:53,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/C/cd1cb539dca04df3b5a2ffa6341cafff as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/cd1cb539dca04df3b5a2ffa6341cafff 2024-11-17T15:37:53,970 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/cd1cb539dca04df3b5a2ffa6341cafff, entries=150, sequenceid=75, filesize=11.7 K 2024-11-17T15:37:53,972 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 5b1940a26f6b6e92cc2f11a0e1deef31 in 881ms, sequenceid=75, compaction requested=false 2024-11-17T15:37:53,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for 5b1940a26f6b6e92cc2f11a0e1deef31: 2024-11-17T15:37:53,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31. 2024-11-17T15:37:53,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-11-17T15:37:53,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-11-17T15:37:53,976 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-11-17T15:37:53,976 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3470 sec 2024-11-17T15:37:53,982 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 1.3540 sec 2024-11-17T15:37:54,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(8855): Flush requested on 5b1940a26f6b6e92cc2f11a0e1deef31 2024-11-17T15:37:54,045 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T15:37:54,045 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5b1940a26f6b6e92cc2f11a0e1deef31 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-17T15:37:54,052 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/A/f8a8c55e77784da49c8ec9c7466fcde2 is 50, key is test_row_0/A:col10/1731857874043/Put/seqid=0 2024-11-17T15:37:54,060 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742087_1263 (size=16681) 2024-11-17T15:37:54,061 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/A/f8a8c55e77784da49c8ec9c7466fcde2 2024-11-17T15:37:54,070 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/B/49cb5d73dbe74f3fbc0551e6f8977fe0 is 50, key is test_row_0/B:col10/1731857874043/Put/seqid=0 2024-11-17T15:37:54,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742088_1264 (size=12001) 2024-11-17T15:37:54,088 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:54,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35954 deadline: 1731857934083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:54,088 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:54,089 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:54,089 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:37:54,090 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:54,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35958 deadline: 1731857934085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:54,090 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:54,090 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:54,090 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:37:54,093 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:54,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35940 deadline: 1731857934088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:54,094 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:54,094 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:54,094 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:54,094 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:37:54,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35884 deadline: 1731857934088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:54,094 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:54,094 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:54,094 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:37:54,196 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:54,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35954 deadline: 1731857934193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:54,196 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:54,197 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:54,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35958 deadline: 1731857934193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:54,197 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:54,197 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:37:54,197 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:54,197 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:54,197 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:37:54,203 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:54,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35940 deadline: 1731857934203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:54,203 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:54,203 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:54,203 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:37:54,207 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:54,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35884 deadline: 1731857934203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:54,208 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:54,208 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:54,208 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:37:54,406 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:54,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35954 deadline: 1731857934403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:54,407 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:54,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35958 deadline: 1731857934403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:54,407 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:54,407 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:54,407 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:37:54,407 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:54,407 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:54,407 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:37:54,416 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:54,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35940 deadline: 1731857934413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:54,416 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:54,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35884 deadline: 1731857934413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:54,416 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:54,416 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:54,416 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:54,417 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:37:54,417 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:54,417 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:37:54,479 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/B/49cb5d73dbe74f3fbc0551e6f8977fe0 2024-11-17T15:37:54,507 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/C/580315bc583e48da9ceff321fd01b1d4 is 50, key is test_row_0/C:col10/1731857874043/Put/seqid=0 2024-11-17T15:37:54,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742089_1265 (size=12001) 2024-11-17T15:37:54,717 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:54,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35954 deadline: 1731857934713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:54,718 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:54,718 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:54,718 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:54,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35958 deadline: 1731857934714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:54,718 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:37:54,718 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:54,718 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:54,718 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:37:54,724 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:54,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35884 deadline: 1731857934723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:54,725 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:54,725 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:54,725 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:37:54,727 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:54,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35940 deadline: 1731857934723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:54,728 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:54,728 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:54,728 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:37:54,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-17T15:37:54,783 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T15:37:54,783 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T15:37:54,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-11-17T15:37:54,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-17T15:37:54,785 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T15:37:54,786 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T15:37:54,786 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T15:37:54,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-17T15:37:54,938 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-17T15:37:54,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31. 2024-11-17T15:37:54,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31. as already flushing 2024-11-17T15:37:54,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31. 2024-11-17T15:37:54,938 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 5b1940a26f6b6e92cc2f11a0e1deef31, NAME => 'TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:37:54,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 5b1940a26f6b6e92cc2f11a0e1deef31, NAME => 'TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:37:54,939 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/C/580315bc583e48da9ceff321fd01b1d4 2024-11-17T15:37:54,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b1940a26f6b6e92cc2f11a0e1deef31, NAME => 'TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b1940a26f6b6e92cc2f11a0e1deef31, NAME => 'TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:37:54,945 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/A/f8a8c55e77784da49c8ec9c7466fcde2 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/f8a8c55e77784da49c8ec9c7466fcde2 2024-11-17T15:37:54,952 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/f8a8c55e77784da49c8ec9c7466fcde2, entries=250, sequenceid=91, filesize=16.3 K 2024-11-17T15:37:54,954 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/B/49cb5d73dbe74f3fbc0551e6f8977fe0 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/49cb5d73dbe74f3fbc0551e6f8977fe0 2024-11-17T15:37:54,959 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/49cb5d73dbe74f3fbc0551e6f8977fe0, entries=150, sequenceid=91, filesize=11.7 K 2024-11-17T15:37:54,961 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/C/580315bc583e48da9ceff321fd01b1d4 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/580315bc583e48da9ceff321fd01b1d4 2024-11-17T15:37:54,966 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/580315bc583e48da9ceff321fd01b1d4, entries=150, sequenceid=91, filesize=11.7 K 2024-11-17T15:37:54,968 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 5b1940a26f6b6e92cc2f11a0e1deef31 in 923ms, sequenceid=91, compaction requested=true 2024-11-17T15:37:54,968 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5b1940a26f6b6e92cc2f11a0e1deef31: 2024-11-17T15:37:54,969 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5b1940a26f6b6e92cc2f11a0e1deef31:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T15:37:54,969 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T15:37:54,970 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T15:37:54,970 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5b1940a26f6b6e92cc2f11a0e1deef31:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T15:37:54,970 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T15:37:54,970 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T15:37:54,971 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5b1940a26f6b6e92cc2f11a0e1deef31:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T15:37:54,971 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T15:37:54,971 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40786 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T15:37:54,971 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1541): 5b1940a26f6b6e92cc2f11a0e1deef31/A is initiating minor compaction (all files) 2024-11-17T15:37:54,971 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5b1940a26f6b6e92cc2f11a0e1deef31/A in TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31. 2024-11-17T15:37:54,971 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/934518f8976a4194ad4f338b0602ffd0, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/000b01d9ad3c462e943c54bd46627c66, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/f8a8c55e77784da49c8ec9c7466fcde2] into tmpdir=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp, totalSize=39.8 K 2024-11-17T15:37:54,972 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T15:37:54,972 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 934518f8976a4194ad4f338b0602ffd0, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731857871728 2024-11-17T15:37:54,972 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1541): 5b1940a26f6b6e92cc2f11a0e1deef31/B is initiating minor compaction (all files) 2024-11-17T15:37:54,972 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5b1940a26f6b6e92cc2f11a0e1deef31/B in TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31. 2024-11-17T15:37:54,972 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/8b53743e04b74406b5e549c1aeeb997c, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/c246c40addd44edfa90e375ff06ad5cb, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/49cb5d73dbe74f3fbc0551e6f8977fe0] into tmpdir=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp, totalSize=35.3 K 2024-11-17T15:37:54,973 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 000b01d9ad3c462e943c54bd46627c66, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1731857871764 2024-11-17T15:37:54,973 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting 8b53743e04b74406b5e549c1aeeb997c, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731857871728 2024-11-17T15:37:54,973 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting f8a8c55e77784da49c8ec9c7466fcde2, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1731857873926 2024-11-17T15:37:54,974 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting c246c40addd44edfa90e375ff06ad5cb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1731857871764 2024-11-17T15:37:54,974 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting 49cb5d73dbe74f3fbc0551e6f8977fe0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1731857873926 2024-11-17T15:37:54,985 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5b1940a26f6b6e92cc2f11a0e1deef31#B#compaction#215 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T15:37:54,986 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/B/3351b845f76e44fba0c5fb014bff1786 is 50, key is test_row_0/B:col10/1731857874043/Put/seqid=0 2024-11-17T15:37:55,002 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5b1940a26f6b6e92cc2f11a0e1deef31#A#compaction#216 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T15:37:55,003 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/A/56b92318691f46c195c13dc1ab1b47ba is 50, key is test_row_0/A:col10/1731857874043/Put/seqid=0 2024-11-17T15:37:55,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742090_1266 (size=12207) 2024-11-17T15:37:55,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742091_1267 (size=12207) 2024-11-17T15:37:55,093 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-17T15:37:55,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31. 2024-11-17T15:37:55,093 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing 5b1940a26f6b6e92cc2f11a0e1deef31 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-17T15:37:55,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/A/00678687ce4a480a8b5b349009361971 is 50, key is test_row_0/A:col10/1731857874078/Put/seqid=0 2024-11-17T15:37:55,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-17T15:37:55,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742092_1268 (size=12001) 2024-11-17T15:37:55,115 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=113 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/A/00678687ce4a480a8b5b349009361971 2024-11-17T15:37:55,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/B/e3e2f723b80c45f886bef34eebad2fa6 is 50, key is test_row_0/B:col10/1731857874078/Put/seqid=0 2024-11-17T15:37:55,133 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742093_1269 (size=12001) 2024-11-17T15:37:55,134 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=113 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/B/e3e2f723b80c45f886bef34eebad2fa6 2024-11-17T15:37:55,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/C/2a9bee46afc2408f95773ca34ea2f3fa is 50, key is test_row_0/C:col10/1731857874078/Put/seqid=0 2024-11-17T15:37:55,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742094_1270 (size=12001) 2024-11-17T15:37:55,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(8855): Flush requested on 5b1940a26f6b6e92cc2f11a0e1deef31 2024-11-17T15:37:55,227 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31. as already flushing 2024-11-17T15:37:55,250 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:55,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35958 deadline: 1731857935244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:55,250 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:55,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35884 deadline: 1731857935245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:55,251 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:55,251 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:55,251 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:55,251 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:55,251 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:37:55,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35940 deadline: 1731857935246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:55,251 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:55,251 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:37:55,251 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:55,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35954 deadline: 1731857935247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:55,251 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:55,251 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:55,251 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:37:55,251 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:55,251 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:55,251 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:37:55,357 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:55,357 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:55,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35940 deadline: 1731857935353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:55,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35958 deadline: 1731857935353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:55,357 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:55,357 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:55,357 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:55,357 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:55,357 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:55,357 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:55,358 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:37:55,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35884 deadline: 1731857935353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:55,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35954 deadline: 1731857935353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:55,358 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:37:55,358 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:55,358 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:55,358 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:55,358 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:55,358 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:37:55,358 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:37:55,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-17T15:37:55,417 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/B/3351b845f76e44fba0c5fb014bff1786 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/3351b845f76e44fba0c5fb014bff1786 2024-11-17T15:37:55,425 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/A/56b92318691f46c195c13dc1ab1b47ba as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/56b92318691f46c195c13dc1ab1b47ba 2024-11-17T15:37:55,426 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5b1940a26f6b6e92cc2f11a0e1deef31/B of 5b1940a26f6b6e92cc2f11a0e1deef31 into 3351b845f76e44fba0c5fb014bff1786(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T15:37:55,426 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5b1940a26f6b6e92cc2f11a0e1deef31: 2024-11-17T15:37:55,426 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., storeName=5b1940a26f6b6e92cc2f11a0e1deef31/B, priority=13, startTime=1731857874970; duration=0sec 2024-11-17T15:37:55,426 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T15:37:55,426 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5b1940a26f6b6e92cc2f11a0e1deef31:B 2024-11-17T15:37:55,426 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T15:37:55,427 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T15:37:55,427 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1541): 5b1940a26f6b6e92cc2f11a0e1deef31/C is initiating minor compaction (all files) 2024-11-17T15:37:55,427 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5b1940a26f6b6e92cc2f11a0e1deef31/C in TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31. 2024-11-17T15:37:55,427 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/8078048adee444c68dda5ee03376915b, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/cd1cb539dca04df3b5a2ffa6341cafff, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/580315bc583e48da9ceff321fd01b1d4] into tmpdir=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp, totalSize=35.3 K 2024-11-17T15:37:55,428 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting 8078048adee444c68dda5ee03376915b, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731857871728 2024-11-17T15:37:55,428 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting cd1cb539dca04df3b5a2ffa6341cafff, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1731857871764 2024-11-17T15:37:55,430 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting 580315bc583e48da9ceff321fd01b1d4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1731857873926 2024-11-17T15:37:55,432 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5b1940a26f6b6e92cc2f11a0e1deef31/A of 5b1940a26f6b6e92cc2f11a0e1deef31 into 56b92318691f46c195c13dc1ab1b47ba(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T15:37:55,432 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5b1940a26f6b6e92cc2f11a0e1deef31: 2024-11-17T15:37:55,432 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., storeName=5b1940a26f6b6e92cc2f11a0e1deef31/A, priority=13, startTime=1731857874968; duration=0sec 2024-11-17T15:37:55,432 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T15:37:55,432 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5b1940a26f6b6e92cc2f11a0e1deef31:A 2024-11-17T15:37:55,439 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5b1940a26f6b6e92cc2f11a0e1deef31#C#compaction#220 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T15:37:55,440 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/C/3cc1d8b097034e398ce8ddc63ae36750 is 50, key is test_row_0/C:col10/1731857874043/Put/seqid=0 2024-11-17T15:37:55,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742095_1271 (size=12207) 2024-11-17T15:37:55,453 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/C/3cc1d8b097034e398ce8ddc63ae36750 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/3cc1d8b097034e398ce8ddc63ae36750 2024-11-17T15:37:55,458 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5b1940a26f6b6e92cc2f11a0e1deef31/C of 5b1940a26f6b6e92cc2f11a0e1deef31 into 3cc1d8b097034e398ce8ddc63ae36750(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T15:37:55,458 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5b1940a26f6b6e92cc2f11a0e1deef31: 2024-11-17T15:37:55,458 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., storeName=5b1940a26f6b6e92cc2f11a0e1deef31/C, priority=13, startTime=1731857874970; duration=0sec 2024-11-17T15:37:55,458 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T15:37:55,458 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5b1940a26f6b6e92cc2f11a0e1deef31:C 2024-11-17T15:37:55,560 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=113 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/C/2a9bee46afc2408f95773ca34ea2f3fa 2024-11-17T15:37:55,565 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:55,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35958 deadline: 1731857935563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:55,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/A/00678687ce4a480a8b5b349009361971 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/00678687ce4a480a8b5b349009361971 2024-11-17T15:37:55,566 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:55,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35884 deadline: 1731857935563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:55,566 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:55,566 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:55,566 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:37:55,566 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:55,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35940 deadline: 1731857935564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:55,566 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:55,566 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:55,566 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:55,566 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:55,566 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:55,566 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:37:55,566 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:37:55,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35954 deadline: 1731857935564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:55,567 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:55,567 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:55,567 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:37:55,570 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/00678687ce4a480a8b5b349009361971, entries=150, sequenceid=113, filesize=11.7 K 2024-11-17T15:37:55,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/B/e3e2f723b80c45f886bef34eebad2fa6 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/e3e2f723b80c45f886bef34eebad2fa6 2024-11-17T15:37:55,575 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/e3e2f723b80c45f886bef34eebad2fa6, entries=150, sequenceid=113, filesize=11.7 K 2024-11-17T15:37:55,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/C/2a9bee46afc2408f95773ca34ea2f3fa as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/2a9bee46afc2408f95773ca34ea2f3fa 2024-11-17T15:37:55,579 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/2a9bee46afc2408f95773ca34ea2f3fa, entries=150, sequenceid=113, filesize=11.7 K 2024-11-17T15:37:55,580 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 5b1940a26f6b6e92cc2f11a0e1deef31 in 487ms, sequenceid=113, compaction requested=false 2024-11-17T15:37:55,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for 5b1940a26f6b6e92cc2f11a0e1deef31: 2024-11-17T15:37:55,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31. 2024-11-17T15:37:55,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-11-17T15:37:55,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-11-17T15:37:55,584 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-11-17T15:37:55,584 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 796 msec 2024-11-17T15:37:55,586 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 802 msec 2024-11-17T15:37:55,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(8855): Flush requested on 5b1940a26f6b6e92cc2f11a0e1deef31 2024-11-17T15:37:55,876 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T15:37:55,876 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5b1940a26f6b6e92cc2f11a0e1deef31 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-17T15:37:55,883 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/A/801889d366804c3d8a0f5aa752bf9586 is 50, key is test_row_0/A:col10/1731857875245/Put/seqid=0 2024-11-17T15:37:55,905 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:55,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35940 deadline: 1731857935901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:55,906 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:55,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35954 deadline: 1731857935902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:55,906 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:55,906 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:55,906 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:37:55,906 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:55,906 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:55,906 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:37:55,907 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:55,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35884 deadline: 1731857935905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:55,907 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:55,908 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:55,908 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:37:55,908 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:55,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35958 deadline: 1731857935906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:55,908 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:55,908 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:55,908 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:37:55,916 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742096_1272 (size=12101) 2024-11-17T15:37:55,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-17T15:37:55,923 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T15:37:55,924 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T15:37:55,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-11-17T15:37:55,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-17T15:37:55,926 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T15:37:55,927 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T15:37:55,927 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T15:37:55,967 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:55,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35902 deadline: 1731857935963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:55,968 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 7a780d55532c,32955,1731857800648 for 'test_row_2' in 5b1940a26f6b6e92cc2f11a0e1deef31 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4196 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T15:37:55,968 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:55,968 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:55,968 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:37:56,016 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:56,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35940 deadline: 1731857936013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:56,017 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:56,017 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:56,017 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:56,017 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:37:56,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35954 deadline: 1731857936013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:56,017 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:56,017 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:56,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35884 deadline: 1731857936013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:56,017 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:56,017 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:37:56,017 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:56,018 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:56,018 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:56,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35958 deadline: 1731857936013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:56,018 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:37:56,018 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:56,018 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:56,018 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:37:56,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-17T15:37:56,079 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-17T15:37:56,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31. 2024-11-17T15:37:56,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31. as already flushing 2024-11-17T15:37:56,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31. 2024-11-17T15:37:56,080 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 5b1940a26f6b6e92cc2f11a0e1deef31, NAME => 'TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:37:56,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 5b1940a26f6b6e92cc2f11a0e1deef31, NAME => 'TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:37:56,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b1940a26f6b6e92cc2f11a0e1deef31, NAME => 'TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b1940a26f6b6e92cc2f11a0e1deef31, NAME => 'TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:37:56,224 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:56,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35940 deadline: 1731857936223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:56,225 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:56,225 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:56,225 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:56,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35954 deadline: 1731857936223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:56,225 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:37:56,225 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:56,225 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:56,225 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:56,225 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:37:56,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35958 deadline: 1731857936224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:56,225 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:56,225 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:56,225 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:56,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35884 deadline: 1731857936224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:56,226 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:37:56,226 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:56,226 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:56,226 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:37:56,233 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-17T15:37:56,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31. 2024-11-17T15:37:56,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31. as already flushing 2024-11-17T15:37:56,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31. 2024-11-17T15:37:56,234 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 5b1940a26f6b6e92cc2f11a0e1deef31, NAME => 'TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:37:56,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 5b1940a26f6b6e92cc2f11a0e1deef31, NAME => 'TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:37:56,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b1940a26f6b6e92cc2f11a0e1deef31, NAME => 'TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b1940a26f6b6e92cc2f11a0e1deef31, NAME => 'TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:37:56,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-17T15:37:56,317 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/A/801889d366804c3d8a0f5aa752bf9586 2024-11-17T15:37:56,326 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/B/e505abe1256f414da8d30356ce7f5ef2 is 50, key is test_row_0/B:col10/1731857875245/Put/seqid=0 2024-11-17T15:37:56,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742097_1273 (size=12101) 2024-11-17T15:37:56,336 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/B/e505abe1256f414da8d30356ce7f5ef2 2024-11-17T15:37:56,345 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/C/ec3c907c70ae46feba90245a7a841d44 is 50, key is test_row_0/C:col10/1731857875245/Put/seqid=0 2024-11-17T15:37:56,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742098_1274 (size=12101) 2024-11-17T15:37:56,356 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/C/ec3c907c70ae46feba90245a7a841d44 2024-11-17T15:37:56,362 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/A/801889d366804c3d8a0f5aa752bf9586 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/801889d366804c3d8a0f5aa752bf9586 2024-11-17T15:37:56,367 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/801889d366804c3d8a0f5aa752bf9586, entries=150, sequenceid=133, filesize=11.8 K 2024-11-17T15:37:56,368 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/B/e505abe1256f414da8d30356ce7f5ef2 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/e505abe1256f414da8d30356ce7f5ef2 2024-11-17T15:37:56,373 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/e505abe1256f414da8d30356ce7f5ef2, entries=150, sequenceid=133, filesize=11.8 K 2024-11-17T15:37:56,374 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/C/ec3c907c70ae46feba90245a7a841d44 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/ec3c907c70ae46feba90245a7a841d44 2024-11-17T15:37:56,378 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/ec3c907c70ae46feba90245a7a841d44, entries=150, sequenceid=133, filesize=11.8 K 2024-11-17T15:37:56,380 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for 5b1940a26f6b6e92cc2f11a0e1deef31 in 503ms, sequenceid=133, compaction requested=true 2024-11-17T15:37:56,380 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5b1940a26f6b6e92cc2f11a0e1deef31: 2024-11-17T15:37:56,380 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5b1940a26f6b6e92cc2f11a0e1deef31:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T15:37:56,380 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T15:37:56,380 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T15:37:56,380 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5b1940a26f6b6e92cc2f11a0e1deef31:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T15:37:56,380 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T15:37:56,380 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5b1940a26f6b6e92cc2f11a0e1deef31:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T15:37:56,380 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T15:37:56,380 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T15:37:56,382 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T15:37:56,382 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1541): 5b1940a26f6b6e92cc2f11a0e1deef31/B is initiating minor compaction (all files) 2024-11-17T15:37:56,382 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5b1940a26f6b6e92cc2f11a0e1deef31/B in TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31. 2024-11-17T15:37:56,382 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/3351b845f76e44fba0c5fb014bff1786, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/e3e2f723b80c45f886bef34eebad2fa6, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/e505abe1256f414da8d30356ce7f5ef2] into tmpdir=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp, totalSize=35.5 K 2024-11-17T15:37:56,382 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T15:37:56,382 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1541): 5b1940a26f6b6e92cc2f11a0e1deef31/A is initiating minor compaction (all files) 2024-11-17T15:37:56,383 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5b1940a26f6b6e92cc2f11a0e1deef31/A in TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31. 2024-11-17T15:37:56,383 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/56b92318691f46c195c13dc1ab1b47ba, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/00678687ce4a480a8b5b349009361971, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/801889d366804c3d8a0f5aa752bf9586] into tmpdir=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp, totalSize=35.5 K 2024-11-17T15:37:56,383 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting 3351b845f76e44fba0c5fb014bff1786, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1731857873926 2024-11-17T15:37:56,384 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 56b92318691f46c195c13dc1ab1b47ba, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1731857873926 2024-11-17T15:37:56,384 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting e3e2f723b80c45f886bef34eebad2fa6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1731857874078 2024-11-17T15:37:56,385 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 00678687ce4a480a8b5b349009361971, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1731857874078 2024-11-17T15:37:56,385 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting e505abe1256f414da8d30356ce7f5ef2, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731857875245 2024-11-17T15:37:56,387 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 801889d366804c3d8a0f5aa752bf9586, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731857875245 2024-11-17T15:37:56,387 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-17T15:37:56,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31. 2024-11-17T15:37:56,388 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing 5b1940a26f6b6e92cc2f11a0e1deef31 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-17T15:37:56,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/A/a6d1a27e618b4a14a89c3f93aa504041 is 50, key is test_row_0/A:col10/1731857875883/Put/seqid=0 2024-11-17T15:37:56,400 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5b1940a26f6b6e92cc2f11a0e1deef31#A#compaction#225 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T15:37:56,401 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/A/bf9d7f64f3704372ac4ecd868f72d203 is 50, key is test_row_0/A:col10/1731857875245/Put/seqid=0 2024-11-17T15:37:56,410 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5b1940a26f6b6e92cc2f11a0e1deef31#B#compaction#226 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T15:37:56,410 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/B/f14a6fec481f4e88a80bd8edf9a28f42 is 50, key is test_row_0/B:col10/1731857875245/Put/seqid=0 2024-11-17T15:37:56,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742099_1275 (size=12151) 2024-11-17T15:37:56,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742100_1276 (size=12409) 2024-11-17T15:37:56,447 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/A/bf9d7f64f3704372ac4ecd868f72d203 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/bf9d7f64f3704372ac4ecd868f72d203 2024-11-17T15:37:56,453 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5b1940a26f6b6e92cc2f11a0e1deef31/A of 5b1940a26f6b6e92cc2f11a0e1deef31 into bf9d7f64f3704372ac4ecd868f72d203(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T15:37:56,454 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5b1940a26f6b6e92cc2f11a0e1deef31: 2024-11-17T15:37:56,454 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., storeName=5b1940a26f6b6e92cc2f11a0e1deef31/A, priority=13, startTime=1731857876380; duration=0sec 2024-11-17T15:37:56,454 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T15:37:56,454 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5b1940a26f6b6e92cc2f11a0e1deef31:A 2024-11-17T15:37:56,454 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T15:37:56,455 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T15:37:56,455 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1541): 5b1940a26f6b6e92cc2f11a0e1deef31/C is initiating minor compaction (all files) 2024-11-17T15:37:56,455 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5b1940a26f6b6e92cc2f11a0e1deef31/C in TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31. 2024-11-17T15:37:56,456 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/3cc1d8b097034e398ce8ddc63ae36750, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/2a9bee46afc2408f95773ca34ea2f3fa, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/ec3c907c70ae46feba90245a7a841d44] into tmpdir=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp, totalSize=35.5 K 2024-11-17T15:37:56,456 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3cc1d8b097034e398ce8ddc63ae36750, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1731857873926 2024-11-17T15:37:56,456 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2a9bee46afc2408f95773ca34ea2f3fa, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1731857874078 2024-11-17T15:37:56,457 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting ec3c907c70ae46feba90245a7a841d44, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731857875245 2024-11-17T15:37:56,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742101_1277 (size=12409) 2024-11-17T15:37:56,481 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5b1940a26f6b6e92cc2f11a0e1deef31#C#compaction#227 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T15:37:56,482 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/C/ea23e89f185b4331b76ea65be73e5257 is 50, key is test_row_0/C:col10/1731857875245/Put/seqid=0 2024-11-17T15:37:56,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742102_1278 (size=12409) 2024-11-17T15:37:56,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(8855): Flush requested on 5b1940a26f6b6e92cc2f11a0e1deef31 2024-11-17T15:37:56,535 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31. as already flushing 2024-11-17T15:37:56,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-17T15:37:56,553 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:56,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35954 deadline: 1731857936547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:56,553 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:56,553 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:56,553 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:37:56,557 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:56,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35940 deadline: 1731857936552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:56,557 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:56,557 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:56,557 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:37:56,559 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:56,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35958 deadline: 1731857936553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:56,560 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:56,559 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:56,560 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:56,560 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:37:56,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35884 deadline: 1731857936554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:56,560 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:56,560 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:56,560 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:37:56,666 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:56,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35954 deadline: 1731857936663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:56,666 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:56,666 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:56,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35940 deadline: 1731857936663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:56,666 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:56,666 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:37:56,666 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:56,666 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:56,667 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:56,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35958 deadline: 1731857936663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:56,667 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:37:56,667 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:56,667 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:56,667 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:56,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35884 deadline: 1731857936663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:56,667 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:37:56,667 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:56,667 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:56,667 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:37:56,836 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=152 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/A/a6d1a27e618b4a14a89c3f93aa504041 2024-11-17T15:37:56,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/B/e74b14a65a894410861606574c2464f4 is 50, key is test_row_0/B:col10/1731857875883/Put/seqid=0 2024-11-17T15:37:56,848 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742103_1279 (size=12151) 2024-11-17T15:37:56,849 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=152 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/B/e74b14a65a894410861606574c2464f4 2024-11-17T15:37:56,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/C/9826019ec99f4819b780bca9e3e9f1d5 is 50, key is test_row_0/C:col10/1731857875883/Put/seqid=0 2024-11-17T15:37:56,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742104_1280 (size=12151) 2024-11-17T15:37:56,870 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/B/f14a6fec481f4e88a80bd8edf9a28f42 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/f14a6fec481f4e88a80bd8edf9a28f42 2024-11-17T15:37:56,874 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:56,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35940 deadline: 1731857936873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:56,875 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:56,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35954 deadline: 1731857936873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:56,875 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:56,875 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:56,875 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:37:56,875 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:56,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35958 deadline: 1731857936873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:56,875 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:56,875 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:56,875 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:56,875 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:37:56,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35884 deadline: 1731857936873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:56,875 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:56,876 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:56,876 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:37:56,876 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:56,876 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:56,876 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:37:56,882 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5b1940a26f6b6e92cc2f11a0e1deef31/B of 5b1940a26f6b6e92cc2f11a0e1deef31 into f14a6fec481f4e88a80bd8edf9a28f42(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T15:37:56,883 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5b1940a26f6b6e92cc2f11a0e1deef31: 2024-11-17T15:37:56,883 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., storeName=5b1940a26f6b6e92cc2f11a0e1deef31/B, priority=13, startTime=1731857876380; duration=0sec 2024-11-17T15:37:56,883 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T15:37:56,883 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5b1940a26f6b6e92cc2f11a0e1deef31:B 2024-11-17T15:37:56,905 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/C/ea23e89f185b4331b76ea65be73e5257 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/ea23e89f185b4331b76ea65be73e5257 2024-11-17T15:37:56,914 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5b1940a26f6b6e92cc2f11a0e1deef31/C of 5b1940a26f6b6e92cc2f11a0e1deef31 into ea23e89f185b4331b76ea65be73e5257(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T15:37:56,914 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5b1940a26f6b6e92cc2f11a0e1deef31: 2024-11-17T15:37:56,914 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., storeName=5b1940a26f6b6e92cc2f11a0e1deef31/C, priority=13, startTime=1731857876380; duration=0sec 2024-11-17T15:37:56,914 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T15:37:56,914 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5b1940a26f6b6e92cc2f11a0e1deef31:C 2024-11-17T15:37:57,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-17T15:37:57,183 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:57,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35940 deadline: 1731857937183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:57,184 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:57,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35954 deadline: 1731857937183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:57,184 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:57,184 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:57,184 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:37:57,184 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:57,184 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:57,184 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:37:57,184 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:57,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35884 deadline: 1731857937183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:57,185 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:57,185 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:57,185 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:57,185 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:37:57,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35958 deadline: 1731857937183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:57,185 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:57,185 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:57,185 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:37:57,265 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=152 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/C/9826019ec99f4819b780bca9e3e9f1d5 2024-11-17T15:37:57,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/A/a6d1a27e618b4a14a89c3f93aa504041 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/a6d1a27e618b4a14a89c3f93aa504041 2024-11-17T15:37:57,274 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/a6d1a27e618b4a14a89c3f93aa504041, entries=150, sequenceid=152, filesize=11.9 K 2024-11-17T15:37:57,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/B/e74b14a65a894410861606574c2464f4 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/e74b14a65a894410861606574c2464f4 2024-11-17T15:37:57,280 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/e74b14a65a894410861606574c2464f4, entries=150, sequenceid=152, filesize=11.9 K 2024-11-17T15:37:57,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/C/9826019ec99f4819b780bca9e3e9f1d5 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/9826019ec99f4819b780bca9e3e9f1d5 2024-11-17T15:37:57,285 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/9826019ec99f4819b780bca9e3e9f1d5, entries=150, sequenceid=152, filesize=11.9 K 2024-11-17T15:37:57,286 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=100.63 KB/103050 for 5b1940a26f6b6e92cc2f11a0e1deef31 in 898ms, sequenceid=152, compaction requested=false 2024-11-17T15:37:57,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for 5b1940a26f6b6e92cc2f11a0e1deef31: 2024-11-17T15:37:57,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31. 2024-11-17T15:37:57,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-11-17T15:37:57,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-11-17T15:37:57,289 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-11-17T15:37:57,289 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3600 sec 2024-11-17T15:37:57,292 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 1.3660 sec 2024-11-17T15:37:57,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(8855): Flush requested on 5b1940a26f6b6e92cc2f11a0e1deef31 2024-11-17T15:37:57,695 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T15:37:57,695 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5b1940a26f6b6e92cc2f11a0e1deef31 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-17T15:37:57,700 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/A/01cdd4e6e2d74f2db15ac9f548579b33 is 50, key is test_row_0/A:col10/1731857877693/Put/seqid=0 2024-11-17T15:37:57,705 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742105_1281 (size=12151) 2024-11-17T15:37:57,716 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:57,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35954 deadline: 1731857937710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:57,716 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:57,716 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:57,716 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:37:57,716 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:57,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35940 deadline: 1731857937713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:57,717 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:57,717 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:57,717 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:37:57,721 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:57,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35884 deadline: 1731857937716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:57,721 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:57,721 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:57,721 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:57,721 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:37:57,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35958 deadline: 1731857937716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:57,721 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:57,721 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:57,721 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:37:57,826 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:57,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35954 deadline: 1731857937823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:57,827 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:57,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35940 deadline: 1731857937823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:57,827 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:57,827 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:57,827 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:37:57,827 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:57,827 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:57,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35884 deadline: 1731857937824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:57,827 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:57,827 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:37:57,827 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:57,827 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:57,827 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:37:57,836 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:57,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35958 deadline: 1731857937833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:57,836 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:57,836 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:57,836 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:37:58,036 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:58,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35954 deadline: 1731857938033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:58,037 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:58,037 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:58,037 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:58,037 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:37:58,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35940 deadline: 1731857938033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:58,037 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:58,037 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:58,037 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:37:58,037 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:58,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35884 deadline: 1731857938033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:58,037 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:58,037 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:58,037 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:37:58,046 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:58,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35958 deadline: 1731857938042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:58,046 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:58,046 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:58,046 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:37:58,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-17T15:37:58,073 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T15:37:58,073 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T15:37:58,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees 2024-11-17T15:37:58,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-17T15:37:58,075 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T15:37:58,076 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T15:37:58,076 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T15:37:58,106 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=174 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/A/01cdd4e6e2d74f2db15ac9f548579b33 2024-11-17T15:37:58,113 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/B/769f413d271248539065c3d90cdc962b is 50, key is test_row_0/B:col10/1731857877693/Put/seqid=0 2024-11-17T15:37:58,127 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742106_1282 (size=12151) 2024-11-17T15:37:58,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-17T15:37:58,229 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-17T15:37:58,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31. 2024-11-17T15:37:58,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31. as already flushing 2024-11-17T15:37:58,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31. 2024-11-17T15:37:58,230 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 5b1940a26f6b6e92cc2f11a0e1deef31, NAME => 'TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:37:58,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 5b1940a26f6b6e92cc2f11a0e1deef31, NAME => 'TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:37:58,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b1940a26f6b6e92cc2f11a0e1deef31, NAME => 'TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b1940a26f6b6e92cc2f11a0e1deef31, NAME => 'TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:37:58,346 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:58,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35954 deadline: 1731857938343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:58,346 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:58,346 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:58,346 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:58,346 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:37:58,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35940 deadline: 1731857938343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:58,347 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:58,347 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:58,347 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:37:58,347 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:58,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35884 deadline: 1731857938343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:58,347 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:58,347 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:58,347 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:37:58,353 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:58,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35958 deadline: 1731857938352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:58,354 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:58,354 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:58,354 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:37:58,383 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-17T15:37:58,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31. 2024-11-17T15:37:58,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31. as already flushing 2024-11-17T15:37:58,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31. 2024-11-17T15:37:58,383 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 5b1940a26f6b6e92cc2f11a0e1deef31, NAME => 'TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:37:58,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 5b1940a26f6b6e92cc2f11a0e1deef31, NAME => 'TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:37:58,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b1940a26f6b6e92cc2f11a0e1deef31, NAME => 'TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b1940a26f6b6e92cc2f11a0e1deef31, NAME => 'TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:37:58,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-17T15:37:58,528 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=174 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/B/769f413d271248539065c3d90cdc962b 2024-11-17T15:37:58,537 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/C/676458318fcc4f498acf17fd48d24627 is 50, key is test_row_0/C:col10/1731857877693/Put/seqid=0 2024-11-17T15:37:58,539 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-17T15:37:58,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31. 2024-11-17T15:37:58,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31. as already flushing 2024-11-17T15:37:58,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31. 2024-11-17T15:37:58,540 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 5b1940a26f6b6e92cc2f11a0e1deef31, NAME => 'TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:37:58,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 5b1940a26f6b6e92cc2f11a0e1deef31, NAME => 'TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:37:58,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b1940a26f6b6e92cc2f11a0e1deef31, NAME => 'TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b1940a26f6b6e92cc2f11a0e1deef31, NAME => 'TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:37:58,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742107_1283 (size=12151) 2024-11-17T15:37:58,694 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-17T15:37:58,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31. 2024-11-17T15:37:58,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31. as already flushing 2024-11-17T15:37:58,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31. 2024-11-17T15:37:58,694 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 5b1940a26f6b6e92cc2f11a0e1deef31, NAME => 'TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:37:58,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 5b1940a26f6b6e92cc2f11a0e1deef31, NAME => 'TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:37:58,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b1940a26f6b6e92cc2f11a0e1deef31, NAME => 'TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b1940a26f6b6e92cc2f11a0e1deef31, NAME => 'TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:37:58,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-17T15:37:58,847 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-17T15:37:58,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31. 2024-11-17T15:37:58,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31. as already flushing 2024-11-17T15:37:58,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31. 2024-11-17T15:37:58,847 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 5b1940a26f6b6e92cc2f11a0e1deef31, NAME => 'TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:37:58,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 5b1940a26f6b6e92cc2f11a0e1deef31, NAME => 'TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:37:58,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b1940a26f6b6e92cc2f11a0e1deef31, NAME => 'TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b1940a26f6b6e92cc2f11a0e1deef31, NAME => 'TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:37:58,856 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:58,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35940 deadline: 1731857938853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:58,856 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:58,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35954 deadline: 1731857938853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:58,857 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:58,857 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:58,857 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:58,857 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:58,857 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:37:58,857 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:58,857 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:37:58,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35884 deadline: 1731857938854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:58,857 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:58,857 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:58,857 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:37:58,866 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:58,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35958 deadline: 1731857938863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:58,867 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:58,867 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:58,867 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:37:58,942 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=174 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/C/676458318fcc4f498acf17fd48d24627 2024-11-17T15:37:58,948 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/A/01cdd4e6e2d74f2db15ac9f548579b33 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/01cdd4e6e2d74f2db15ac9f548579b33 2024-11-17T15:37:58,952 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/01cdd4e6e2d74f2db15ac9f548579b33, entries=150, sequenceid=174, filesize=11.9 K 2024-11-17T15:37:58,953 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/B/769f413d271248539065c3d90cdc962b as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/769f413d271248539065c3d90cdc962b 2024-11-17T15:37:58,957 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/769f413d271248539065c3d90cdc962b, entries=150, sequenceid=174, filesize=11.9 K 2024-11-17T15:37:58,958 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/C/676458318fcc4f498acf17fd48d24627 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/676458318fcc4f498acf17fd48d24627 2024-11-17T15:37:58,962 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/676458318fcc4f498acf17fd48d24627, entries=150, sequenceid=174, filesize=11.9 K 2024-11-17T15:37:58,963 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=100.63 KB/103050 for 5b1940a26f6b6e92cc2f11a0e1deef31 in 1268ms, sequenceid=174, compaction requested=true 2024-11-17T15:37:58,964 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5b1940a26f6b6e92cc2f11a0e1deef31: 2024-11-17T15:37:58,964 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5b1940a26f6b6e92cc2f11a0e1deef31:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T15:37:58,964 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T15:37:58,964 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T15:37:58,964 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T15:37:58,964 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5b1940a26f6b6e92cc2f11a0e1deef31:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T15:37:58,964 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T15:37:58,964 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5b1940a26f6b6e92cc2f11a0e1deef31:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T15:37:58,964 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T15:37:58,965 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T15:37:58,965 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1541): 5b1940a26f6b6e92cc2f11a0e1deef31/A is initiating minor compaction (all files) 2024-11-17T15:37:58,965 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5b1940a26f6b6e92cc2f11a0e1deef31/A in TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31. 2024-11-17T15:37:58,965 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/bf9d7f64f3704372ac4ecd868f72d203, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/a6d1a27e618b4a14a89c3f93aa504041, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/01cdd4e6e2d74f2db15ac9f548579b33] into tmpdir=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp, totalSize=35.9 K 2024-11-17T15:37:58,966 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting bf9d7f64f3704372ac4ecd868f72d203, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731857875245 2024-11-17T15:37:58,967 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T15:37:58,967 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting a6d1a27e618b4a14a89c3f93aa504041, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1731857875883 2024-11-17T15:37:58,967 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1541): 5b1940a26f6b6e92cc2f11a0e1deef31/B is initiating minor compaction (all files) 2024-11-17T15:37:58,967 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5b1940a26f6b6e92cc2f11a0e1deef31/B in TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31. 2024-11-17T15:37:58,967 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/f14a6fec481f4e88a80bd8edf9a28f42, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/e74b14a65a894410861606574c2464f4, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/769f413d271248539065c3d90cdc962b] into tmpdir=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp, totalSize=35.9 K 2024-11-17T15:37:58,969 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting f14a6fec481f4e88a80bd8edf9a28f42, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731857875245 2024-11-17T15:37:58,969 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 01cdd4e6e2d74f2db15ac9f548579b33, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1731857876544 2024-11-17T15:37:58,969 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting e74b14a65a894410861606574c2464f4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1731857875883 2024-11-17T15:37:58,970 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting 769f413d271248539065c3d90cdc962b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1731857876544 2024-11-17T15:37:58,991 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5b1940a26f6b6e92cc2f11a0e1deef31#A#compaction#233 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T15:37:58,991 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5b1940a26f6b6e92cc2f11a0e1deef31#B#compaction#234 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T15:37:58,991 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/A/25ccb88d14b443b7bd5bb47fde721ec3 is 50, key is test_row_0/A:col10/1731857877693/Put/seqid=0 2024-11-17T15:37:58,991 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/B/a3474d1a240542ada73649b5e027640f is 50, key is test_row_0/B:col10/1731857877693/Put/seqid=0 2024-11-17T15:37:59,000 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-17T15:37:59,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31. 2024-11-17T15:37:59,001 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2902): Flushing 5b1940a26f6b6e92cc2f11a0e1deef31 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-17T15:37:59,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/A/87210e380159459f92ca022dafac0037 is 50, key is test_row_0/A:col10/1731857877714/Put/seqid=0 2024-11-17T15:37:59,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742108_1284 (size=12561) 2024-11-17T15:37:59,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742109_1285 (size=12561) 2024-11-17T15:37:59,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742110_1286 (size=12151) 2024-11-17T15:37:59,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-17T15:37:59,423 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/B/a3474d1a240542ada73649b5e027640f as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/a3474d1a240542ada73649b5e027640f 2024-11-17T15:37:59,424 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/A/25ccb88d14b443b7bd5bb47fde721ec3 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/25ccb88d14b443b7bd5bb47fde721ec3 2024-11-17T15:37:59,427 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/A/87210e380159459f92ca022dafac0037 2024-11-17T15:37:59,429 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5b1940a26f6b6e92cc2f11a0e1deef31/B of 5b1940a26f6b6e92cc2f11a0e1deef31 into a3474d1a240542ada73649b5e027640f(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T15:37:59,429 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5b1940a26f6b6e92cc2f11a0e1deef31/A of 5b1940a26f6b6e92cc2f11a0e1deef31 into 25ccb88d14b443b7bd5bb47fde721ec3(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T15:37:59,429 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5b1940a26f6b6e92cc2f11a0e1deef31: 2024-11-17T15:37:59,429 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5b1940a26f6b6e92cc2f11a0e1deef31: 2024-11-17T15:37:59,429 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., storeName=5b1940a26f6b6e92cc2f11a0e1deef31/A, priority=13, startTime=1731857878964; duration=0sec 2024-11-17T15:37:59,429 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., storeName=5b1940a26f6b6e92cc2f11a0e1deef31/B, priority=13, startTime=1731857878964; duration=0sec 2024-11-17T15:37:59,429 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T15:37:59,429 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5b1940a26f6b6e92cc2f11a0e1deef31:A 2024-11-17T15:37:59,429 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T15:37:59,429 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5b1940a26f6b6e92cc2f11a0e1deef31:B 2024-11-17T15:37:59,429 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T15:37:59,432 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T15:37:59,432 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1541): 5b1940a26f6b6e92cc2f11a0e1deef31/C is initiating minor compaction (all files) 2024-11-17T15:37:59,432 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5b1940a26f6b6e92cc2f11a0e1deef31/C in TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31. 2024-11-17T15:37:59,432 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/ea23e89f185b4331b76ea65be73e5257, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/9826019ec99f4819b780bca9e3e9f1d5, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/676458318fcc4f498acf17fd48d24627] into tmpdir=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp, totalSize=35.9 K 2024-11-17T15:37:59,432 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting ea23e89f185b4331b76ea65be73e5257, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731857875245 2024-11-17T15:37:59,433 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9826019ec99f4819b780bca9e3e9f1d5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1731857875883 2024-11-17T15:37:59,433 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 676458318fcc4f498acf17fd48d24627, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1731857876544 2024-11-17T15:37:59,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/B/9b94c9f0ab59447ebd9f9da27b06cd54 is 50, key is test_row_0/B:col10/1731857877714/Put/seqid=0 2024-11-17T15:37:59,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742111_1287 (size=12151) 2024-11-17T15:37:59,442 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5b1940a26f6b6e92cc2f11a0e1deef31#C#compaction#237 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T15:37:59,442 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/B/9b94c9f0ab59447ebd9f9da27b06cd54 2024-11-17T15:37:59,443 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/C/dc62506d55194b09b619b00225574893 is 50, key is test_row_0/C:col10/1731857877693/Put/seqid=0 2024-11-17T15:37:59,451 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742112_1288 (size=12561) 2024-11-17T15:37:59,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/C/83fa1df6dd19428087caf41da89ecffc is 50, key is test_row_0/C:col10/1731857877714/Put/seqid=0 2024-11-17T15:37:59,460 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742113_1289 (size=12151) 2024-11-17T15:37:59,460 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/C/83fa1df6dd19428087caf41da89ecffc 2024-11-17T15:37:59,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/A/87210e380159459f92ca022dafac0037 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/87210e380159459f92ca022dafac0037 2024-11-17T15:37:59,470 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/87210e380159459f92ca022dafac0037, entries=150, sequenceid=192, filesize=11.9 K 2024-11-17T15:37:59,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/B/9b94c9f0ab59447ebd9f9da27b06cd54 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/9b94c9f0ab59447ebd9f9da27b06cd54 2024-11-17T15:37:59,475 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/9b94c9f0ab59447ebd9f9da27b06cd54, entries=150, sequenceid=192, filesize=11.9 K 2024-11-17T15:37:59,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/C/83fa1df6dd19428087caf41da89ecffc as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/83fa1df6dd19428087caf41da89ecffc 2024-11-17T15:37:59,481 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/83fa1df6dd19428087caf41da89ecffc, entries=150, sequenceid=192, filesize=11.9 K 2024-11-17T15:37:59,483 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=0 B/0 for 5b1940a26f6b6e92cc2f11a0e1deef31 in 482ms, sequenceid=192, compaction requested=false 2024-11-17T15:37:59,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2603): Flush status journal for 5b1940a26f6b6e92cc2f11a0e1deef31: 2024-11-17T15:37:59,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31. 2024-11-17T15:37:59,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=77 2024-11-17T15:37:59,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.HMaster(4169): Remote procedure done, pid=77 2024-11-17T15:37:59,488 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-11-17T15:37:59,488 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4090 sec 2024-11-17T15:37:59,491 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees in 1.4160 sec 2024-11-17T15:37:59,857 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/C/dc62506d55194b09b619b00225574893 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/dc62506d55194b09b619b00225574893 2024-11-17T15:37:59,863 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5b1940a26f6b6e92cc2f11a0e1deef31/C of 5b1940a26f6b6e92cc2f11a0e1deef31 into dc62506d55194b09b619b00225574893(size=12.3 K), total size for store is 24.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T15:37:59,863 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5b1940a26f6b6e92cc2f11a0e1deef31: 2024-11-17T15:37:59,863 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., storeName=5b1940a26f6b6e92cc2f11a0e1deef31/C, priority=13, startTime=1731857878964; duration=0sec 2024-11-17T15:37:59,864 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T15:37:59,864 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5b1940a26f6b6e92cc2f11a0e1deef31:C 2024-11-17T15:37:59,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(8855): Flush requested on 5b1940a26f6b6e92cc2f11a0e1deef31 2024-11-17T15:37:59,877 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T15:37:59,878 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5b1940a26f6b6e92cc2f11a0e1deef31 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-17T15:37:59,883 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/A/2f5e6813c4ee477e9b64609b25bb4ec9 is 50, key is test_row_0/A:col10/1731857879875/Put/seqid=0 2024-11-17T15:37:59,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742114_1290 (size=14541) 2024-11-17T15:37:59,891 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/A/2f5e6813c4ee477e9b64609b25bb4ec9 2024-11-17T15:37:59,899 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/B/c339b3c0ad10441d893bc2ea050619eb is 50, key is test_row_0/B:col10/1731857879875/Put/seqid=0 2024-11-17T15:37:59,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742115_1291 (size=12151) 2024-11-17T15:37:59,909 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:59,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35940 deadline: 1731857939903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:59,910 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:59,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35884 deadline: 1731857939907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:59,910 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:59,910 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:59,910 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:37:59,910 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:59,910 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:59,910 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:37:59,913 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:59,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35954 deadline: 1731857939909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:59,913 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:37:59,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35958 deadline: 1731857939910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:37:59,914 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:59,914 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:59,914 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:37:59,914 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:59,914 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:37:59,914 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:38:00,004 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:00,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35902 deadline: 1731857940003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:00,005 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 7a780d55532c,32955,1731857800648 for 'test_row_2' in 5b1940a26f6b6e92cc2f11a0e1deef31 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8234 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T15:38:00,006 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:00,006 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:00,006 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:38:00,016 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:00,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35940 deadline: 1731857940013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:00,016 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:00,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35884 deadline: 1731857940013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:00,017 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:00,017 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:00,017 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:00,017 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:38:00,017 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:00,017 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:38:00,026 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:00,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35954 deadline: 1731857940023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:00,026 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:00,026 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:00,026 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:00,026 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:38:00,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35958 deadline: 1731857940023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:00,026 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:00,026 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:00,026 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:38:00,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-17T15:38:00,223 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T15:38:00,224 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T15:38:00,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees 2024-11-17T15:38:00,226 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:00,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-17T15:38:00,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35884 deadline: 1731857940223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:00,226 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T15:38:00,226 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:00,226 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:00,226 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:00,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35940 deadline: 1731857940223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:00,226 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:38:00,227 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T15:38:00,227 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T15:38:00,227 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:00,227 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:00,227 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:38:00,237 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:00,237 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:00,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35954 deadline: 1731857940233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:00,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35958 deadline: 1731857940233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:00,238 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:00,238 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:00,238 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:38:00,238 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:00,238 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:00,238 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:38:00,306 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/B/c339b3c0ad10441d893bc2ea050619eb 2024-11-17T15:38:00,314 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/C/4c35d3bc40a441eb9883282183f70c5f is 50, key is test_row_0/C:col10/1731857879875/Put/seqid=0 2024-11-17T15:38:00,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742116_1292 (size=12151) 2024-11-17T15:38:00,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-17T15:38:00,379 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-17T15:38:00,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31. 2024-11-17T15:38:00,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31. as already flushing 2024-11-17T15:38:00,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31. 2024-11-17T15:38:00,380 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 5b1940a26f6b6e92cc2f11a0e1deef31, NAME => 'TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:00,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 5b1940a26f6b6e92cc2f11a0e1deef31, NAME => 'TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:00,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b1940a26f6b6e92cc2f11a0e1deef31, NAME => 'TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b1940a26f6b6e92cc2f11a0e1deef31, NAME => 'TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:00,533 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-17T15:38:00,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31. 2024-11-17T15:38:00,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31. as already flushing 2024-11-17T15:38:00,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31. 2024-11-17T15:38:00,534 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 5b1940a26f6b6e92cc2f11a0e1deef31, NAME => 'TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:00,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 5b1940a26f6b6e92cc2f11a0e1deef31, NAME => 'TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:00,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b1940a26f6b6e92cc2f11a0e1deef31, NAME => 'TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b1940a26f6b6e92cc2f11a0e1deef31, NAME => 'TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:00,535 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:00,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35940 deadline: 1731857940533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:00,536 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:00,536 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:00,536 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:00,536 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:38:00,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35884 deadline: 1731857940533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:00,536 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:00,536 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:00,536 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:38:00,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-17T15:38:00,546 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:00,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35954 deadline: 1731857940543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:00,547 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:00,547 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:00,547 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:00,547 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:38:00,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35958 deadline: 1731857940543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:00,547 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:00,547 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:00,547 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:38:00,686 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-17T15:38:00,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31. 2024-11-17T15:38:00,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31. as already flushing 2024-11-17T15:38:00,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31. 2024-11-17T15:38:00,686 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 5b1940a26f6b6e92cc2f11a0e1deef31, NAME => 'TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:00,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 5b1940a26f6b6e92cc2f11a0e1deef31, NAME => 'TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:00,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b1940a26f6b6e92cc2f11a0e1deef31, NAME => 'TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b1940a26f6b6e92cc2f11a0e1deef31, NAME => 'TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:00,720 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/C/4c35d3bc40a441eb9883282183f70c5f 2024-11-17T15:38:00,724 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/A/2f5e6813c4ee477e9b64609b25bb4ec9 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/2f5e6813c4ee477e9b64609b25bb4ec9 2024-11-17T15:38:00,729 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/2f5e6813c4ee477e9b64609b25bb4ec9, entries=200, sequenceid=207, filesize=14.2 K 2024-11-17T15:38:00,729 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/B/c339b3c0ad10441d893bc2ea050619eb as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/c339b3c0ad10441d893bc2ea050619eb 2024-11-17T15:38:00,733 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/c339b3c0ad10441d893bc2ea050619eb, entries=150, sequenceid=207, filesize=11.9 K 2024-11-17T15:38:00,734 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/C/4c35d3bc40a441eb9883282183f70c5f as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/4c35d3bc40a441eb9883282183f70c5f 2024-11-17T15:38:00,737 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/4c35d3bc40a441eb9883282183f70c5f, entries=150, sequenceid=207, filesize=11.9 K 2024-11-17T15:38:00,738 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 5b1940a26f6b6e92cc2f11a0e1deef31 in 861ms, sequenceid=207, compaction requested=true 2024-11-17T15:38:00,738 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5b1940a26f6b6e92cc2f11a0e1deef31: 2024-11-17T15:38:00,738 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5b1940a26f6b6e92cc2f11a0e1deef31:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T15:38:00,738 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T15:38:00,738 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5b1940a26f6b6e92cc2f11a0e1deef31:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T15:38:00,739 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T15:38:00,739 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T15:38:00,739 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5b1940a26f6b6e92cc2f11a0e1deef31:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T15:38:00,739 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T15:38:00,739 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T15:38:00,739 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39253 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T15:38:00,740 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T15:38:00,740 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1541): 5b1940a26f6b6e92cc2f11a0e1deef31/A is initiating minor compaction (all files) 2024-11-17T15:38:00,740 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1541): 5b1940a26f6b6e92cc2f11a0e1deef31/B is initiating minor compaction (all files) 2024-11-17T15:38:00,740 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5b1940a26f6b6e92cc2f11a0e1deef31/A in TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31. 2024-11-17T15:38:00,740 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5b1940a26f6b6e92cc2f11a0e1deef31/B in TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31. 2024-11-17T15:38:00,740 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/a3474d1a240542ada73649b5e027640f, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/9b94c9f0ab59447ebd9f9da27b06cd54, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/c339b3c0ad10441d893bc2ea050619eb] into tmpdir=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp, totalSize=36.0 K 2024-11-17T15:38:00,740 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/25ccb88d14b443b7bd5bb47fde721ec3, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/87210e380159459f92ca022dafac0037, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/2f5e6813c4ee477e9b64609b25bb4ec9] into tmpdir=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp, totalSize=38.3 K 2024-11-17T15:38:00,740 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 25ccb88d14b443b7bd5bb47fde721ec3, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1731857876544 2024-11-17T15:38:00,740 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting a3474d1a240542ada73649b5e027640f, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1731857876544 2024-11-17T15:38:00,740 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting 9b94c9f0ab59447ebd9f9da27b06cd54, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1731857877709 2024-11-17T15:38:00,741 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 87210e380159459f92ca022dafac0037, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1731857877709 2024-11-17T15:38:00,741 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2f5e6813c4ee477e9b64609b25bb4ec9, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1731857879866 2024-11-17T15:38:00,741 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting c339b3c0ad10441d893bc2ea050619eb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1731857879866 2024-11-17T15:38:00,749 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5b1940a26f6b6e92cc2f11a0e1deef31#B#compaction#243 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T15:38:00,749 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5b1940a26f6b6e92cc2f11a0e1deef31#A#compaction#242 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T15:38:00,750 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/B/9e8f5e2c82e041af9dec3725f0319242 is 50, key is test_row_0/B:col10/1731857879875/Put/seqid=0 2024-11-17T15:38:00,751 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/A/207de946c0ba451aaacd41426e69f06b is 50, key is test_row_0/A:col10/1731857879875/Put/seqid=0 2024-11-17T15:38:00,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742117_1293 (size=12663) 2024-11-17T15:38:00,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742118_1294 (size=12663) 2024-11-17T15:38:00,763 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/A/207de946c0ba451aaacd41426e69f06b as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/207de946c0ba451aaacd41426e69f06b 2024-11-17T15:38:00,770 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5b1940a26f6b6e92cc2f11a0e1deef31/A of 5b1940a26f6b6e92cc2f11a0e1deef31 into 207de946c0ba451aaacd41426e69f06b(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T15:38:00,770 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5b1940a26f6b6e92cc2f11a0e1deef31: 2024-11-17T15:38:00,770 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., storeName=5b1940a26f6b6e92cc2f11a0e1deef31/A, priority=13, startTime=1731857880738; duration=0sec 2024-11-17T15:38:00,770 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T15:38:00,770 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5b1940a26f6b6e92cc2f11a0e1deef31:A 2024-11-17T15:38:00,770 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T15:38:00,771 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T15:38:00,771 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1541): 5b1940a26f6b6e92cc2f11a0e1deef31/C is initiating minor compaction (all files) 2024-11-17T15:38:00,771 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5b1940a26f6b6e92cc2f11a0e1deef31/C in TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31. 2024-11-17T15:38:00,772 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/dc62506d55194b09b619b00225574893, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/83fa1df6dd19428087caf41da89ecffc, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/4c35d3bc40a441eb9883282183f70c5f] into tmpdir=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp, totalSize=36.0 K 2024-11-17T15:38:00,772 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting dc62506d55194b09b619b00225574893, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1731857876544 2024-11-17T15:38:00,772 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 83fa1df6dd19428087caf41da89ecffc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1731857877709 2024-11-17T15:38:00,773 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4c35d3bc40a441eb9883282183f70c5f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1731857879866 2024-11-17T15:38:00,780 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5b1940a26f6b6e92cc2f11a0e1deef31#C#compaction#244 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T15:38:00,781 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/C/afbe04b1dcd2492c96a8e74200dceeb5 is 50, key is test_row_0/C:col10/1731857879875/Put/seqid=0 2024-11-17T15:38:00,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742119_1295 (size=12663) 2024-11-17T15:38:00,839 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-17T15:38:00,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31. 2024-11-17T15:38:00,839 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2902): Flushing 5b1940a26f6b6e92cc2f11a0e1deef31 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-17T15:38:00,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/A/99278c6b91204675b6862a60f044e29f is 50, key is test_row_0/A:col10/1731857879907/Put/seqid=0 2024-11-17T15:38:00,848 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742120_1296 (size=12151) 2024-11-17T15:38:00,849 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=232 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/A/99278c6b91204675b6862a60f044e29f 2024-11-17T15:38:00,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-17T15:38:00,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/B/4efdd0e1ddd2439cb59429a3e4c4dbf3 is 50, key is test_row_0/B:col10/1731857879907/Put/seqid=0 2024-11-17T15:38:00,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742121_1297 (size=12151) 2024-11-17T15:38:01,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(8855): Flush requested on 5b1940a26f6b6e92cc2f11a0e1deef31 2024-11-17T15:38:01,045 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31. as already flushing 2024-11-17T15:38:01,061 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:01,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35884 deadline: 1731857941057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:01,062 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:01,062 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:01,062 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:01,062 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:38:01,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35954 deadline: 1731857941057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:01,062 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:01,062 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:01,062 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:38:01,062 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:01,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35958 deadline: 1731857941058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:01,063 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:01,063 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:01,063 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:38:01,066 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:01,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35940 deadline: 1731857941062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:01,067 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:01,067 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:01,067 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:38:01,162 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/B/9e8f5e2c82e041af9dec3725f0319242 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/9e8f5e2c82e041af9dec3725f0319242 2024-11-17T15:38:01,168 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5b1940a26f6b6e92cc2f11a0e1deef31/B of 5b1940a26f6b6e92cc2f11a0e1deef31 into 9e8f5e2c82e041af9dec3725f0319242(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T15:38:01,168 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5b1940a26f6b6e92cc2f11a0e1deef31: 2024-11-17T15:38:01,168 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., storeName=5b1940a26f6b6e92cc2f11a0e1deef31/B, priority=13, startTime=1731857880738; duration=0sec 2024-11-17T15:38:01,168 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T15:38:01,168 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5b1940a26f6b6e92cc2f11a0e1deef31:B 2024-11-17T15:38:01,176 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:01,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35954 deadline: 1731857941173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:01,176 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:01,176 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:01,176 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:38:01,177 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:01,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35958 deadline: 1731857941173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:01,177 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:01,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35940 deadline: 1731857941173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:01,178 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:01,178 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:01,178 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:01,178 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:38:01,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35884 deadline: 1731857941173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:01,178 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:01,178 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:01,178 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:38:01,188 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:01,188 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:01,188 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:38:01,200 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/C/afbe04b1dcd2492c96a8e74200dceeb5 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/afbe04b1dcd2492c96a8e74200dceeb5 2024-11-17T15:38:01,204 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5b1940a26f6b6e92cc2f11a0e1deef31/C of 5b1940a26f6b6e92cc2f11a0e1deef31 into afbe04b1dcd2492c96a8e74200dceeb5(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T15:38:01,204 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5b1940a26f6b6e92cc2f11a0e1deef31: 2024-11-17T15:38:01,204 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., storeName=5b1940a26f6b6e92cc2f11a0e1deef31/C, priority=13, startTime=1731857880739; duration=0sec 2024-11-17T15:38:01,204 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T15:38:01,204 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5b1940a26f6b6e92cc2f11a0e1deef31:C 2024-11-17T15:38:01,264 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=232 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/B/4efdd0e1ddd2439cb59429a3e4c4dbf3 2024-11-17T15:38:01,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/C/6edfcfa84d5245538fdb2377b787b93d is 50, key is test_row_0/C:col10/1731857879907/Put/seqid=0 2024-11-17T15:38:01,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742122_1298 (size=12151) 2024-11-17T15:38:01,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-17T15:38:01,386 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:01,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35940 deadline: 1731857941383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:01,387 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:01,387 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:01,387 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:01,387 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:38:01,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35884 deadline: 1731857941383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:01,387 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:01,387 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:01,387 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:01,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35954 deadline: 1731857941383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:01,387 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:38:01,387 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:01,387 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:01,387 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:38:01,396 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:01,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35958 deadline: 1731857941393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:01,397 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:01,397 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:01,397 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:38:01,678 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=232 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/C/6edfcfa84d5245538fdb2377b787b93d 2024-11-17T15:38:01,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/A/99278c6b91204675b6862a60f044e29f as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/99278c6b91204675b6862a60f044e29f 2024-11-17T15:38:01,690 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/99278c6b91204675b6862a60f044e29f, entries=150, sequenceid=232, filesize=11.9 K 2024-11-17T15:38:01,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/B/4efdd0e1ddd2439cb59429a3e4c4dbf3 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/4efdd0e1ddd2439cb59429a3e4c4dbf3 2024-11-17T15:38:01,696 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:01,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35940 deadline: 1731857941693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:01,696 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/4efdd0e1ddd2439cb59429a3e4c4dbf3, entries=150, sequenceid=232, filesize=11.9 K 2024-11-17T15:38:01,696 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:01,696 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:01,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35884 deadline: 1731857941693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:01,696 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:01,696 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:38:01,696 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:01,697 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:01,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35954 deadline: 1731857941693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:01,697 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:01,697 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:38:01,697 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:01,697 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:01,697 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:38:01,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/C/6edfcfa84d5245538fdb2377b787b93d as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/6edfcfa84d5245538fdb2377b787b93d 2024-11-17T15:38:01,701 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/6edfcfa84d5245538fdb2377b787b93d, entries=150, sequenceid=232, filesize=11.9 K 2024-11-17T15:38:01,702 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 5b1940a26f6b6e92cc2f11a0e1deef31 in 863ms, sequenceid=232, compaction requested=false 2024-11-17T15:38:01,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2603): Flush status journal for 5b1940a26f6b6e92cc2f11a0e1deef31: 2024-11-17T15:38:01,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31. 2024-11-17T15:38:01,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=79 2024-11-17T15:38:01,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.HMaster(4169): Remote procedure done, pid=79 2024-11-17T15:38:01,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(8855): Flush requested on 5b1940a26f6b6e92cc2f11a0e1deef31 2024-11-17T15:38:01,705 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T15:38:01,705 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5b1940a26f6b6e92cc2f11a0e1deef31 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-17T15:38:01,708 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-11-17T15:38:01,708 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4780 sec 2024-11-17T15:38:01,710 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees in 1.4850 sec 2024-11-17T15:38:01,711 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/A/ae878d7290814d9482c7d16a82fcafba is 50, key is test_row_0/A:col10/1731857881055/Put/seqid=0 2024-11-17T15:38:01,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742123_1299 (size=12151) 2024-11-17T15:38:01,808 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:01,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35958 deadline: 1731857941805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:01,809 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:01,809 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:01,809 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:38:01,914 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:01,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35958 deadline: 1731857941913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:01,914 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:01,914 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:01,914 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:38:02,117 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/A/ae878d7290814d9482c7d16a82fcafba 2024-11-17T15:38:02,124 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/B/e977c8abd3654b8aaa5babe13427bf0d is 50, key is test_row_0/B:col10/1731857881055/Put/seqid=0 2024-11-17T15:38:02,125 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:02,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35958 deadline: 1731857942123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:02,126 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:02,126 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:02,126 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:38:02,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742124_1300 (size=12151) 2024-11-17T15:38:02,206 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:02,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35940 deadline: 1731857942203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:02,206 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:02,207 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:02,207 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:02,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35884 deadline: 1731857942203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:02,207 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:38:02,207 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:02,207 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:02,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35954 deadline: 1731857942203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:02,207 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:02,207 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:38:02,207 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:02,207 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:02,207 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:38:02,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-17T15:38:02,373 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T15:38:02,373 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T15:38:02,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] procedure2.ProcedureExecutor(1139): Stored pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees 2024-11-17T15:38:02,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-17T15:38:02,375 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T15:38:02,376 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T15:38:02,376 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T15:38:02,437 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:02,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35958 deadline: 1731857942433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:02,437 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:02,437 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:02,437 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:38:02,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-17T15:38:02,529 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-17T15:38:02,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31. 2024-11-17T15:38:02,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31. as already flushing 2024-11-17T15:38:02,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31. 2024-11-17T15:38:02,529 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 5b1940a26f6b6e92cc2f11a0e1deef31, NAME => 'TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:02,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 5b1940a26f6b6e92cc2f11a0e1deef31, NAME => 'TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:02,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b1940a26f6b6e92cc2f11a0e1deef31, NAME => 'TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b1940a26f6b6e92cc2f11a0e1deef31, NAME => 'TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:02,533 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/B/e977c8abd3654b8aaa5babe13427bf0d 2024-11-17T15:38:02,540 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/C/003cdf15f8394f2eb9bf67feb72a80c4 is 50, key is test_row_0/C:col10/1731857881055/Put/seqid=0 2024-11-17T15:38:02,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742125_1301 (size=12151) 2024-11-17T15:38:02,682 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-17T15:38:02,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31. 2024-11-17T15:38:02,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31. as already flushing 2024-11-17T15:38:02,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31. 2024-11-17T15:38:02,682 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 5b1940a26f6b6e92cc2f11a0e1deef31, NAME => 'TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:02,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 5b1940a26f6b6e92cc2f11a0e1deef31, NAME => 'TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:02,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b1940a26f6b6e92cc2f11a0e1deef31, NAME => 'TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b1940a26f6b6e92cc2f11a0e1deef31, NAME => 'TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:02,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-17T15:38:02,835 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-17T15:38:02,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31. 2024-11-17T15:38:02,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31. as already flushing 2024-11-17T15:38:02,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31. 2024-11-17T15:38:02,836 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 5b1940a26f6b6e92cc2f11a0e1deef31, NAME => 'TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:02,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 5b1940a26f6b6e92cc2f11a0e1deef31, NAME => 'TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:02,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b1940a26f6b6e92cc2f11a0e1deef31, NAME => 'TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b1940a26f6b6e92cc2f11a0e1deef31, NAME => 'TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:02,946 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:02,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35958 deadline: 1731857942943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:02,947 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:02,947 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:02,947 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:38:02,959 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/C/003cdf15f8394f2eb9bf67feb72a80c4 2024-11-17T15:38:02,964 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/A/ae878d7290814d9482c7d16a82fcafba as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/ae878d7290814d9482c7d16a82fcafba 2024-11-17T15:38:02,973 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/ae878d7290814d9482c7d16a82fcafba, entries=150, sequenceid=247, filesize=11.9 K 2024-11-17T15:38:02,974 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/B/e977c8abd3654b8aaa5babe13427bf0d as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/e977c8abd3654b8aaa5babe13427bf0d 2024-11-17T15:38:02,978 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/e977c8abd3654b8aaa5babe13427bf0d, entries=150, sequenceid=247, filesize=11.9 K 2024-11-17T15:38:02,979 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/C/003cdf15f8394f2eb9bf67feb72a80c4 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/003cdf15f8394f2eb9bf67feb72a80c4 2024-11-17T15:38:02,982 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/003cdf15f8394f2eb9bf67feb72a80c4, entries=150, sequenceid=247, filesize=11.9 K 2024-11-17T15:38:02,983 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 5b1940a26f6b6e92cc2f11a0e1deef31 in 1278ms, sequenceid=247, compaction requested=true 2024-11-17T15:38:02,983 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5b1940a26f6b6e92cc2f11a0e1deef31: 2024-11-17T15:38:02,983 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5b1940a26f6b6e92cc2f11a0e1deef31:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T15:38:02,983 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T15:38:02,984 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T15:38:02,984 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T15:38:02,984 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5b1940a26f6b6e92cc2f11a0e1deef31:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T15:38:02,984 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T15:38:02,984 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5b1940a26f6b6e92cc2f11a0e1deef31:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T15:38:02,984 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T15:38:02,985 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T15:38:02,985 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1541): 5b1940a26f6b6e92cc2f11a0e1deef31/A is initiating minor compaction (all files) 2024-11-17T15:38:02,985 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5b1940a26f6b6e92cc2f11a0e1deef31/A in TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31. 2024-11-17T15:38:02,985 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/207de946c0ba451aaacd41426e69f06b, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/99278c6b91204675b6862a60f044e29f, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/ae878d7290814d9482c7d16a82fcafba] into tmpdir=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp, totalSize=36.1 K 2024-11-17T15:38:02,985 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T15:38:02,985 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1541): 5b1940a26f6b6e92cc2f11a0e1deef31/B is initiating minor compaction (all files) 2024-11-17T15:38:02,985 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5b1940a26f6b6e92cc2f11a0e1deef31/B in TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31. 2024-11-17T15:38:02,985 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/9e8f5e2c82e041af9dec3725f0319242, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/4efdd0e1ddd2439cb59429a3e4c4dbf3, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/e977c8abd3654b8aaa5babe13427bf0d] into tmpdir=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp, totalSize=36.1 K 2024-11-17T15:38:02,985 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 207de946c0ba451aaacd41426e69f06b, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1731857879866 2024-11-17T15:38:02,986 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting 9e8f5e2c82e041af9dec3725f0319242, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1731857879866 2024-11-17T15:38:02,986 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 99278c6b91204675b6862a60f044e29f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1731857879902 2024-11-17T15:38:02,986 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting 4efdd0e1ddd2439cb59429a3e4c4dbf3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1731857879902 2024-11-17T15:38:02,986 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting ae878d7290814d9482c7d16a82fcafba, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1731857881055 2024-11-17T15:38:02,986 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting e977c8abd3654b8aaa5babe13427bf0d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1731857881055 2024-11-17T15:38:02,987 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-17T15:38:02,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31. 2024-11-17T15:38:02,988 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2902): Flushing 5b1940a26f6b6e92cc2f11a0e1deef31 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-17T15:38:02,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/A/0ecda31f47b549b9b77afb68e7c6a551 is 50, key is test_row_0/A:col10/1731857881798/Put/seqid=0 2024-11-17T15:38:03,002 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5b1940a26f6b6e92cc2f11a0e1deef31#B#compaction#252 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T15:38:03,002 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/B/71a6304c2d3c474ab274f840d7f60d63 is 50, key is test_row_0/B:col10/1731857881055/Put/seqid=0 2024-11-17T15:38:03,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-17T15:38:03,014 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5b1940a26f6b6e92cc2f11a0e1deef31#A#compaction#253 average throughput is 0.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T15:38:03,016 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/A/fb73ad213044429bb361c49b5f5b05a4 is 50, key is test_row_0/A:col10/1731857881055/Put/seqid=0 2024-11-17T15:38:03,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742126_1302 (size=12301) 2024-11-17T15:38:03,019 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=270 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/A/0ecda31f47b549b9b77afb68e7c6a551 2024-11-17T15:38:03,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742128_1304 (size=12765) 2024-11-17T15:38:03,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742127_1303 (size=12765) 2024-11-17T15:38:03,039 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/B/71a6304c2d3c474ab274f840d7f60d63 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/71a6304c2d3c474ab274f840d7f60d63 2024-11-17T15:38:03,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/B/aea528ceb29543a484127fd6b8d2d0b0 is 50, key is test_row_0/B:col10/1731857881798/Put/seqid=0 2024-11-17T15:38:03,046 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5b1940a26f6b6e92cc2f11a0e1deef31/B of 5b1940a26f6b6e92cc2f11a0e1deef31 into 71a6304c2d3c474ab274f840d7f60d63(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T15:38:03,046 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5b1940a26f6b6e92cc2f11a0e1deef31: 2024-11-17T15:38:03,046 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., storeName=5b1940a26f6b6e92cc2f11a0e1deef31/B, priority=13, startTime=1731857882983; duration=0sec 2024-11-17T15:38:03,046 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T15:38:03,046 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5b1940a26f6b6e92cc2f11a0e1deef31:B 2024-11-17T15:38:03,046 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T15:38:03,048 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T15:38:03,048 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1541): 5b1940a26f6b6e92cc2f11a0e1deef31/C is initiating minor compaction (all files) 2024-11-17T15:38:03,048 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5b1940a26f6b6e92cc2f11a0e1deef31/C in TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31. 2024-11-17T15:38:03,048 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/afbe04b1dcd2492c96a8e74200dceeb5, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/6edfcfa84d5245538fdb2377b787b93d, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/003cdf15f8394f2eb9bf67feb72a80c4] into tmpdir=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp, totalSize=36.1 K 2024-11-17T15:38:03,049 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting afbe04b1dcd2492c96a8e74200dceeb5, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1731857879866 2024-11-17T15:38:03,052 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting 6edfcfa84d5245538fdb2377b787b93d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1731857879902 2024-11-17T15:38:03,052 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting 003cdf15f8394f2eb9bf67feb72a80c4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1731857881055 2024-11-17T15:38:03,069 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5b1940a26f6b6e92cc2f11a0e1deef31#C#compaction#255 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T15:38:03,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742129_1305 (size=12301) 2024-11-17T15:38:03,070 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/C/7242f6fc6ec548dba2f2ef0babcfdc47 is 50, key is test_row_0/C:col10/1731857881055/Put/seqid=0 2024-11-17T15:38:03,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742130_1306 (size=12765) 2024-11-17T15:38:03,083 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/C/7242f6fc6ec548dba2f2ef0babcfdc47 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/7242f6fc6ec548dba2f2ef0babcfdc47 2024-11-17T15:38:03,088 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5b1940a26f6b6e92cc2f11a0e1deef31/C of 5b1940a26f6b6e92cc2f11a0e1deef31 into 7242f6fc6ec548dba2f2ef0babcfdc47(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T15:38:03,088 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5b1940a26f6b6e92cc2f11a0e1deef31: 2024-11-17T15:38:03,088 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., storeName=5b1940a26f6b6e92cc2f11a0e1deef31/C, priority=13, startTime=1731857882984; duration=0sec 2024-11-17T15:38:03,089 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T15:38:03,089 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5b1940a26f6b6e92cc2f11a0e1deef31:C 2024-11-17T15:38:03,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(8855): Flush requested on 5b1940a26f6b6e92cc2f11a0e1deef31 2024-11-17T15:38:03,216 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31. as already flushing 2024-11-17T15:38:03,234 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:03,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35940 deadline: 1731857943229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:03,235 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:03,235 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:03,235 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:38:03,235 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:03,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35884 deadline: 1731857943231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:03,236 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:03,236 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:03,236 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:38:03,240 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:03,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35954 deadline: 1731857943234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:03,240 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:03,240 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:03,240 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:38:03,347 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:03,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35940 deadline: 1731857943343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:03,347 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:03,347 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:03,347 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:03,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35884 deadline: 1731857943343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:03,347 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:38:03,347 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:03,348 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:03,347 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:03,348 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:38:03,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35954 deadline: 1731857943343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:03,348 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:03,348 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:03,348 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:38:03,435 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/A/fb73ad213044429bb361c49b5f5b05a4 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/fb73ad213044429bb361c49b5f5b05a4 2024-11-17T15:38:03,442 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5b1940a26f6b6e92cc2f11a0e1deef31/A of 5b1940a26f6b6e92cc2f11a0e1deef31 into fb73ad213044429bb361c49b5f5b05a4(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T15:38:03,442 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5b1940a26f6b6e92cc2f11a0e1deef31: 2024-11-17T15:38:03,442 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., storeName=5b1940a26f6b6e92cc2f11a0e1deef31/A, priority=13, startTime=1731857882983; duration=0sec 2024-11-17T15:38:03,442 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T15:38:03,442 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5b1940a26f6b6e92cc2f11a0e1deef31:A 2024-11-17T15:38:03,470 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=270 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/B/aea528ceb29543a484127fd6b8d2d0b0 2024-11-17T15:38:03,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/C/87f0885598b2493985ebad87d49e4658 is 50, key is test_row_0/C:col10/1731857881798/Put/seqid=0 2024-11-17T15:38:03,523 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742131_1307 (size=12301) 2024-11-17T15:38:03,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-17T15:38:03,556 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:03,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35940 deadline: 1731857943553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:03,557 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:03,557 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:03,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35884 deadline: 1731857943553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:03,557 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:03,557 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:38:03,557 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:03,557 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:03,557 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:03,557 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:38:03,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35954 deadline: 1731857943553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:03,558 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:03,558 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:03,558 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:38:03,865 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:03,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35940 deadline: 1731857943863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:03,866 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:03,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35954 deadline: 1731857943863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:03,866 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:03,866 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:03,866 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:38:03,866 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:03,866 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:03,866 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:03,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35884 deadline: 1731857943863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:03,866 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:38:03,866 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:03,866 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:03,866 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:38:03,924 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=270 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/C/87f0885598b2493985ebad87d49e4658 2024-11-17T15:38:03,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/A/0ecda31f47b549b9b77afb68e7c6a551 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/0ecda31f47b549b9b77afb68e7c6a551 2024-11-17T15:38:03,934 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/0ecda31f47b549b9b77afb68e7c6a551, entries=150, sequenceid=270, filesize=12.0 K 2024-11-17T15:38:03,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/B/aea528ceb29543a484127fd6b8d2d0b0 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/aea528ceb29543a484127fd6b8d2d0b0 2024-11-17T15:38:03,939 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/aea528ceb29543a484127fd6b8d2d0b0, entries=150, sequenceid=270, filesize=12.0 K 2024-11-17T15:38:03,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/C/87f0885598b2493985ebad87d49e4658 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/87f0885598b2493985ebad87d49e4658 2024-11-17T15:38:03,945 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/87f0885598b2493985ebad87d49e4658, entries=150, sequenceid=270, filesize=12.0 K 2024-11-17T15:38:03,947 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 5b1940a26f6b6e92cc2f11a0e1deef31 in 959ms, sequenceid=270, compaction requested=false 2024-11-17T15:38:03,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2603): Flush status journal for 5b1940a26f6b6e92cc2f11a0e1deef31: 2024-11-17T15:38:03,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31. 2024-11-17T15:38:03,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=81 2024-11-17T15:38:03,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.HMaster(4169): Remote procedure done, pid=81 2024-11-17T15:38:03,950 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-11-17T15:38:03,950 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5720 sec 2024-11-17T15:38:03,953 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees in 1.5780 sec 2024-11-17T15:38:03,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(8855): Flush requested on 5b1940a26f6b6e92cc2f11a0e1deef31 2024-11-17T15:38:03,955 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T15:38:03,955 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5b1940a26f6b6e92cc2f11a0e1deef31 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-17T15:38:03,960 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/A/b1f0eb77d5bb4746a099e42ec88e7cc6 is 50, key is test_row_0/A:col10/1731857883225/Put/seqid=0 2024-11-17T15:38:03,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742132_1308 (size=12301) 2024-11-17T15:38:03,965 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=287 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/A/b1f0eb77d5bb4746a099e42ec88e7cc6 2024-11-17T15:38:03,979 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/B/bae8c5325a9d40a5a7658c9b4549082b is 50, key is test_row_0/B:col10/1731857883225/Put/seqid=0 2024-11-17T15:38:03,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742133_1309 (size=12301) 2024-11-17T15:38:03,985 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=287 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/B/bae8c5325a9d40a5a7658c9b4549082b 2024-11-17T15:38:03,993 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/C/9178f5d18d664cf583be44f411f675ae is 50, key is test_row_0/C:col10/1731857883225/Put/seqid=0 2024-11-17T15:38:03,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742134_1310 (size=12301) 2024-11-17T15:38:03,997 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=287 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/C/9178f5d18d664cf583be44f411f675ae 2024-11-17T15:38:04,002 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/A/b1f0eb77d5bb4746a099e42ec88e7cc6 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/b1f0eb77d5bb4746a099e42ec88e7cc6 2024-11-17T15:38:04,007 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/b1f0eb77d5bb4746a099e42ec88e7cc6, entries=150, sequenceid=287, filesize=12.0 K 2024-11-17T15:38:04,008 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/B/bae8c5325a9d40a5a7658c9b4549082b as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/bae8c5325a9d40a5a7658c9b4549082b 2024-11-17T15:38:04,011 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/bae8c5325a9d40a5a7658c9b4549082b, entries=150, sequenceid=287, filesize=12.0 K 2024-11-17T15:38:04,012 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/C/9178f5d18d664cf583be44f411f675ae as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/9178f5d18d664cf583be44f411f675ae 2024-11-17T15:38:04,017 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/9178f5d18d664cf583be44f411f675ae, entries=150, sequenceid=287, filesize=12.0 K 2024-11-17T15:38:04,018 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=93.93 KB/96180 for 5b1940a26f6b6e92cc2f11a0e1deef31 in 63ms, sequenceid=287, compaction requested=true 2024-11-17T15:38:04,018 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5b1940a26f6b6e92cc2f11a0e1deef31: 2024-11-17T15:38:04,018 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5b1940a26f6b6e92cc2f11a0e1deef31:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T15:38:04,018 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T15:38:04,018 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T15:38:04,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(8855): Flush requested on 5b1940a26f6b6e92cc2f11a0e1deef31 2024-11-17T15:38:04,018 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5b1940a26f6b6e92cc2f11a0e1deef31:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T15:38:04,019 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T15:38:04,019 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T15:38:04,019 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5b1940a26f6b6e92cc2f11a0e1deef31:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T15:38:04,019 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T15:38:04,019 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T15:38:04,019 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5b1940a26f6b6e92cc2f11a0e1deef31 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-17T15:38:04,020 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37367 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T15:38:04,020 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37367 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T15:38:04,020 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1541): 5b1940a26f6b6e92cc2f11a0e1deef31/B is initiating minor compaction (all files) 2024-11-17T15:38:04,020 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1541): 5b1940a26f6b6e92cc2f11a0e1deef31/A is initiating minor compaction (all files) 2024-11-17T15:38:04,020 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5b1940a26f6b6e92cc2f11a0e1deef31/B in TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31. 2024-11-17T15:38:04,020 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5b1940a26f6b6e92cc2f11a0e1deef31/A in TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31. 2024-11-17T15:38:04,020 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/71a6304c2d3c474ab274f840d7f60d63, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/aea528ceb29543a484127fd6b8d2d0b0, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/bae8c5325a9d40a5a7658c9b4549082b] into tmpdir=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp, totalSize=36.5 K 2024-11-17T15:38:04,020 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/fb73ad213044429bb361c49b5f5b05a4, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/0ecda31f47b549b9b77afb68e7c6a551, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/b1f0eb77d5bb4746a099e42ec88e7cc6] into tmpdir=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp, totalSize=36.5 K 2024-11-17T15:38:04,020 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting 71a6304c2d3c474ab274f840d7f60d63, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1731857881055 2024-11-17T15:38:04,020 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting fb73ad213044429bb361c49b5f5b05a4, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1731857881055 2024-11-17T15:38:04,020 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting aea528ceb29543a484127fd6b8d2d0b0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1731857881793 2024-11-17T15:38:04,021 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0ecda31f47b549b9b77afb68e7c6a551, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1731857881793 2024-11-17T15:38:04,021 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting bae8c5325a9d40a5a7658c9b4549082b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1731857883225 2024-11-17T15:38:04,021 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting b1f0eb77d5bb4746a099e42ec88e7cc6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1731857883225 2024-11-17T15:38:04,028 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/A/f1c2836660494d6684f1a65a8124c808 is 50, key is test_row_0/A:col10/1731857883993/Put/seqid=0 2024-11-17T15:38:04,044 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5b1940a26f6b6e92cc2f11a0e1deef31#B#compaction#261 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T15:38:04,045 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/B/9c4842868a6b45188c986515c379efed is 50, key is test_row_0/B:col10/1731857883225/Put/seqid=0 2024-11-17T15:38:04,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742135_1311 (size=14741) 2024-11-17T15:38:04,054 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5b1940a26f6b6e92cc2f11a0e1deef31#A#compaction#262 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T15:38:04,055 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/A/48c7cc7e10a34a7db823c17730b818fa is 50, key is test_row_0/A:col10/1731857883225/Put/seqid=0 2024-11-17T15:38:04,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742136_1312 (size=13017) 2024-11-17T15:38:04,060 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742137_1313 (size=13017) 2024-11-17T15:38:04,065 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/A/48c7cc7e10a34a7db823c17730b818fa as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/48c7cc7e10a34a7db823c17730b818fa 2024-11-17T15:38:04,071 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5b1940a26f6b6e92cc2f11a0e1deef31/A of 5b1940a26f6b6e92cc2f11a0e1deef31 into 48c7cc7e10a34a7db823c17730b818fa(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T15:38:04,071 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5b1940a26f6b6e92cc2f11a0e1deef31: 2024-11-17T15:38:04,071 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., storeName=5b1940a26f6b6e92cc2f11a0e1deef31/A, priority=13, startTime=1731857884018; duration=0sec 2024-11-17T15:38:04,071 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T15:38:04,071 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5b1940a26f6b6e92cc2f11a0e1deef31:A 2024-11-17T15:38:04,071 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T15:38:04,072 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37367 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T15:38:04,072 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1541): 5b1940a26f6b6e92cc2f11a0e1deef31/C is initiating minor compaction (all files) 2024-11-17T15:38:04,072 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5b1940a26f6b6e92cc2f11a0e1deef31/C in TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31. 2024-11-17T15:38:04,072 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/7242f6fc6ec548dba2f2ef0babcfdc47, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/87f0885598b2493985ebad87d49e4658, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/9178f5d18d664cf583be44f411f675ae] into tmpdir=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp, totalSize=36.5 K 2024-11-17T15:38:04,072 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7242f6fc6ec548dba2f2ef0babcfdc47, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1731857881055 2024-11-17T15:38:04,073 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 87f0885598b2493985ebad87d49e4658, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1731857881793 2024-11-17T15:38:04,073 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9178f5d18d664cf583be44f411f675ae, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1731857883225 2024-11-17T15:38:04,083 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5b1940a26f6b6e92cc2f11a0e1deef31#C#compaction#263 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T15:38:04,083 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/C/eca894e019e4423f9b31f52367577268 is 50, key is test_row_0/C:col10/1731857883225/Put/seqid=0 2024-11-17T15:38:04,109 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:04,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35958 deadline: 1731857944103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:04,110 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:04,110 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:04,110 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:38:04,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742138_1314 (size=13017) 2024-11-17T15:38:04,126 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/C/eca894e019e4423f9b31f52367577268 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/eca894e019e4423f9b31f52367577268 2024-11-17T15:38:04,131 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5b1940a26f6b6e92cc2f11a0e1deef31/C of 5b1940a26f6b6e92cc2f11a0e1deef31 into eca894e019e4423f9b31f52367577268(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T15:38:04,131 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5b1940a26f6b6e92cc2f11a0e1deef31: 2024-11-17T15:38:04,131 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., storeName=5b1940a26f6b6e92cc2f11a0e1deef31/C, priority=13, startTime=1731857884019; duration=0sec 2024-11-17T15:38:04,131 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T15:38:04,131 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5b1940a26f6b6e92cc2f11a0e1deef31:C 2024-11-17T15:38:04,216 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:04,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35958 deadline: 1731857944213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:04,216 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:04,216 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:04,216 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:38:04,376 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:04,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35940 deadline: 1731857944373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:04,377 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:04,377 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:04,377 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:04,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35954 deadline: 1731857944373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:04,377 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:38:04,377 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:04,377 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:04,377 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:04,377 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:38:04,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35884 deadline: 1731857944373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:04,378 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:04,378 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:04,378 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:38:04,426 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:04,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35958 deadline: 1731857944423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:04,426 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:04,426 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:04,426 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:38:04,450 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=305 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/A/f1c2836660494d6684f1a65a8124c808 2024-11-17T15:38:04,458 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/B/7d8d3eba323c4583b0e62cfe15a6d22a is 50, key is test_row_0/B:col10/1731857883993/Put/seqid=0 2024-11-17T15:38:04,461 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/B/9c4842868a6b45188c986515c379efed as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/9c4842868a6b45188c986515c379efed 2024-11-17T15:38:04,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742139_1315 (size=12301) 2024-11-17T15:38:04,463 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=305 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/B/7d8d3eba323c4583b0e62cfe15a6d22a 2024-11-17T15:38:04,468 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5b1940a26f6b6e92cc2f11a0e1deef31/B of 5b1940a26f6b6e92cc2f11a0e1deef31 into 9c4842868a6b45188c986515c379efed(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T15:38:04,468 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5b1940a26f6b6e92cc2f11a0e1deef31: 2024-11-17T15:38:04,468 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., storeName=5b1940a26f6b6e92cc2f11a0e1deef31/B, priority=13, startTime=1731857884018; duration=0sec 2024-11-17T15:38:04,468 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T15:38:04,468 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5b1940a26f6b6e92cc2f11a0e1deef31:B 2024-11-17T15:38:04,472 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/C/d4b48d87873f4f7a8a383622b5c5822e is 50, key is test_row_0/C:col10/1731857883993/Put/seqid=0 2024-11-17T15:38:04,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742140_1316 (size=12301) 2024-11-17T15:38:04,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-17T15:38:04,533 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T15:38:04,534 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T15:38:04,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] procedure2.ProcedureExecutor(1139): Stored pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees 2024-11-17T15:38:04,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-17T15:38:04,536 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T15:38:04,537 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T15:38:04,537 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T15:38:04,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-17T15:38:04,689 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-17T15:38:04,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31. 2024-11-17T15:38:04,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31. as already flushing 2024-11-17T15:38:04,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31. 2024-11-17T15:38:04,690 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 5b1940a26f6b6e92cc2f11a0e1deef31, NAME => 'TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:04,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 5b1940a26f6b6e92cc2f11a0e1deef31, NAME => 'TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:04,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b1940a26f6b6e92cc2f11a0e1deef31, NAME => 'TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b1940a26f6b6e92cc2f11a0e1deef31, NAME => 'TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:04,735 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:04,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35958 deadline: 1731857944733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:04,736 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:04,736 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:04,736 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:38:04,842 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-17T15:38:04,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31. 2024-11-17T15:38:04,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31. as already flushing 2024-11-17T15:38:04,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31. 2024-11-17T15:38:04,843 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 5b1940a26f6b6e92cc2f11a0e1deef31, NAME => 'TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:04,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 5b1940a26f6b6e92cc2f11a0e1deef31, NAME => 'TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:04,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b1940a26f6b6e92cc2f11a0e1deef31, NAME => 'TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b1940a26f6b6e92cc2f11a0e1deef31, NAME => 'TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:04,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-17T15:38:04,877 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=305 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/C/d4b48d87873f4f7a8a383622b5c5822e 2024-11-17T15:38:04,882 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/A/f1c2836660494d6684f1a65a8124c808 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/f1c2836660494d6684f1a65a8124c808 2024-11-17T15:38:04,885 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/f1c2836660494d6684f1a65a8124c808, entries=200, sequenceid=305, filesize=14.4 K 2024-11-17T15:38:04,886 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/B/7d8d3eba323c4583b0e62cfe15a6d22a as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/7d8d3eba323c4583b0e62cfe15a6d22a 2024-11-17T15:38:04,897 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/7d8d3eba323c4583b0e62cfe15a6d22a, entries=150, sequenceid=305, filesize=12.0 K 2024-11-17T15:38:04,898 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/C/d4b48d87873f4f7a8a383622b5c5822e as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/d4b48d87873f4f7a8a383622b5c5822e 2024-11-17T15:38:04,902 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/d4b48d87873f4f7a8a383622b5c5822e, entries=150, sequenceid=305, filesize=12.0 K 2024-11-17T15:38:04,902 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for 5b1940a26f6b6e92cc2f11a0e1deef31 in 883ms, sequenceid=305, compaction requested=false 2024-11-17T15:38:04,903 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5b1940a26f6b6e92cc2f11a0e1deef31: 2024-11-17T15:38:04,996 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-17T15:38:04,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31. 2024-11-17T15:38:04,996 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2902): Flushing 5b1940a26f6b6e92cc2f11a0e1deef31 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-17T15:38:05,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/A/050b11bf94d945ae95ccf95263eefe19 is 50, key is test_row_0/A:col10/1731857884081/Put/seqid=0 2024-11-17T15:38:05,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742141_1317 (size=12301) 2024-11-17T15:38:05,006 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/A/050b11bf94d945ae95ccf95263eefe19 2024-11-17T15:38:05,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/B/4c4b281e03f242c89611ba2d5a1965f6 is 50, key is test_row_0/B:col10/1731857884081/Put/seqid=0 2024-11-17T15:38:05,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742142_1318 (size=12301) 2024-11-17T15:38:05,022 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/B/4c4b281e03f242c89611ba2d5a1965f6 2024-11-17T15:38:05,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/C/12c3f61c3ee1488c9b0aeb6fb923268b is 50, key is test_row_0/C:col10/1731857884081/Put/seqid=0 2024-11-17T15:38:05,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742143_1319 (size=12301) 2024-11-17T15:38:05,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-17T15:38:05,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(8855): Flush requested on 5b1940a26f6b6e92cc2f11a0e1deef31 2024-11-17T15:38:05,246 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31. as already flushing 2024-11-17T15:38:05,321 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:05,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35958 deadline: 1731857945317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:05,321 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:05,321 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:05,321 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:38:05,386 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:05,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35954 deadline: 1731857945383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:05,386 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 7a780d55532c,32955,1731857800648 for 'test_row_1' in 5b1940a26f6b6e92cc2f11a0e1deef31 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2152 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T15:38:05,387 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:05,387 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:05,387 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:38:05,396 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:05,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35940 deadline: 1731857945393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:05,397 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:05,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35884 deadline: 1731857945393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:05,397 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 7a780d55532c,32955,1731857800648 for 'test_row_2' in 5b1940a26f6b6e92cc2f11a0e1deef31 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2168 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T15:38:05,397 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:05,397 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:05,397 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:38:05,397 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 7a780d55532c,32955,1731857800648 for 'test_row_2' in 5b1940a26f6b6e92cc2f11a0e1deef31 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2166 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T15:38:05,398 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:05,398 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:05,398 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:38:05,424 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:05,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35958 deadline: 1731857945423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:05,425 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:05,425 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:05,425 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:38:05,435 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/C/12c3f61c3ee1488c9b0aeb6fb923268b 2024-11-17T15:38:05,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/A/050b11bf94d945ae95ccf95263eefe19 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/050b11bf94d945ae95ccf95263eefe19 2024-11-17T15:38:05,443 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/050b11bf94d945ae95ccf95263eefe19, entries=150, sequenceid=326, filesize=12.0 K 2024-11-17T15:38:05,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/B/4c4b281e03f242c89611ba2d5a1965f6 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/4c4b281e03f242c89611ba2d5a1965f6 2024-11-17T15:38:05,448 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/4c4b281e03f242c89611ba2d5a1965f6, entries=150, sequenceid=326, filesize=12.0 K 2024-11-17T15:38:05,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/C/12c3f61c3ee1488c9b0aeb6fb923268b as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/12c3f61c3ee1488c9b0aeb6fb923268b 2024-11-17T15:38:05,452 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/12c3f61c3ee1488c9b0aeb6fb923268b, entries=150, sequenceid=326, filesize=12.0 K 2024-11-17T15:38:05,453 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for 5b1940a26f6b6e92cc2f11a0e1deef31 in 457ms, sequenceid=326, compaction requested=true 2024-11-17T15:38:05,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2603): Flush status journal for 5b1940a26f6b6e92cc2f11a0e1deef31: 2024-11-17T15:38:05,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31. 2024-11-17T15:38:05,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=83 2024-11-17T15:38:05,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.HMaster(4169): Remote procedure done, pid=83 2024-11-17T15:38:05,457 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-11-17T15:38:05,457 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 918 msec 2024-11-17T15:38:05,459 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees in 924 msec 2024-11-17T15:38:05,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(8855): Flush requested on 5b1940a26f6b6e92cc2f11a0e1deef31 2024-11-17T15:38:05,636 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T15:38:05,636 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5b1940a26f6b6e92cc2f11a0e1deef31 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-17T15:38:05,640 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/A/e8d27a803ea24454b72758c25b02ef64 is 50, key is test_row_0/A:col10/1731857885303/Put/seqid=0 2024-11-17T15:38:05,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742144_1320 (size=14741) 2024-11-17T15:38:05,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-17T15:38:05,673 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T15:38:05,673 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T15:38:05,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] procedure2.ProcedureExecutor(1139): Stored pid=84, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees 2024-11-17T15:38:05,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-17T15:38:05,675 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=84, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T15:38:05,676 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=84, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T15:38:05,676 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T15:38:05,705 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:05,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35958 deadline: 1731857945700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:05,705 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:05,705 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:05,705 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:38:05,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-17T15:38:05,814 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:05,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35958 deadline: 1731857945813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:05,815 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:05,815 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:05,815 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:38:05,828 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-17T15:38:05,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31. 2024-11-17T15:38:05,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31. as already flushing 2024-11-17T15:38:05,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31. 2024-11-17T15:38:05,829 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => 5b1940a26f6b6e92cc2f11a0e1deef31, NAME => 'TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:05,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => 5b1940a26f6b6e92cc2f11a0e1deef31, NAME => 'TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:05,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b1940a26f6b6e92cc2f11a0e1deef31, NAME => 'TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b1940a26f6b6e92cc2f11a0e1deef31, NAME => 'TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:05,982 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-17T15:38:05,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31. 2024-11-17T15:38:05,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31. as already flushing 2024-11-17T15:38:05,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31. 2024-11-17T15:38:05,982 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => 5b1940a26f6b6e92cc2f11a0e1deef31, NAME => 'TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:05,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => 5b1940a26f6b6e92cc2f11a0e1deef31, NAME => 'TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:05,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b1940a26f6b6e92cc2f11a0e1deef31, NAME => 'TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b1940a26f6b6e92cc2f11a0e1deef31, NAME => 'TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:05,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-17T15:38:06,026 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:06,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35958 deadline: 1731857946023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:06,026 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:06,027 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:06,027 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:38:06,047 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=345 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/A/e8d27a803ea24454b72758c25b02ef64 2024-11-17T15:38:06,055 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/B/2b109fdc1665474594783adc4a515f56 is 50, key is test_row_0/B:col10/1731857885303/Put/seqid=0 2024-11-17T15:38:06,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742145_1321 (size=12301) 2024-11-17T15:38:06,135 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-17T15:38:06,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31. 2024-11-17T15:38:06,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31. as already flushing 2024-11-17T15:38:06,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31. 2024-11-17T15:38:06,136 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => 5b1940a26f6b6e92cc2f11a0e1deef31, NAME => 'TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:06,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => 5b1940a26f6b6e92cc2f11a0e1deef31, NAME => 'TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:06,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b1940a26f6b6e92cc2f11a0e1deef31, NAME => 'TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b1940a26f6b6e92cc2f11a0e1deef31, NAME => 'TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:06,289 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-17T15:38:06,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31. 2024-11-17T15:38:06,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31. as already flushing 2024-11-17T15:38:06,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31. 2024-11-17T15:38:06,289 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => 5b1940a26f6b6e92cc2f11a0e1deef31, NAME => 'TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:06,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => 5b1940a26f6b6e92cc2f11a0e1deef31, NAME => 'TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:06,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b1940a26f6b6e92cc2f11a0e1deef31, NAME => 'TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b1940a26f6b6e92cc2f11a0e1deef31, NAME => 'TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:06,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-17T15:38:06,337 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:06,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35958 deadline: 1731857946333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:06,337 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:06,337 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:06,337 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:38:06,442 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-17T15:38:06,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31. 2024-11-17T15:38:06,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31. as already flushing 2024-11-17T15:38:06,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31. 2024-11-17T15:38:06,442 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => 5b1940a26f6b6e92cc2f11a0e1deef31, NAME => 'TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:06,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => 5b1940a26f6b6e92cc2f11a0e1deef31, NAME => 'TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:06,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b1940a26f6b6e92cc2f11a0e1deef31, NAME => 'TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b1940a26f6b6e92cc2f11a0e1deef31, NAME => 'TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:06,459 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=345 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/B/2b109fdc1665474594783adc4a515f56 2024-11-17T15:38:06,467 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/C/4ddba6187bc143ca9fcfc71c90f06b44 is 50, key is test_row_0/C:col10/1731857885303/Put/seqid=0 2024-11-17T15:38:06,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742146_1322 (size=12301) 2024-11-17T15:38:06,595 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-17T15:38:06,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31. 2024-11-17T15:38:06,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31. as already flushing 2024-11-17T15:38:06,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31. 2024-11-17T15:38:06,595 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => 5b1940a26f6b6e92cc2f11a0e1deef31, NAME => 'TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:06,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => 5b1940a26f6b6e92cc2f11a0e1deef31, NAME => 'TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:06,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b1940a26f6b6e92cc2f11a0e1deef31, NAME => 'TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b1940a26f6b6e92cc2f11a0e1deef31, NAME => 'TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:06,748 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-17T15:38:06,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31. 2024-11-17T15:38:06,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31. as already flushing 2024-11-17T15:38:06,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31. 2024-11-17T15:38:06,749 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => 5b1940a26f6b6e92cc2f11a0e1deef31, NAME => 'TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:06,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => 5b1940a26f6b6e92cc2f11a0e1deef31, NAME => 'TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:06,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b1940a26f6b6e92cc2f11a0e1deef31, NAME => 'TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b1940a26f6b6e92cc2f11a0e1deef31, NAME => 'TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:06,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-17T15:38:06,846 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:06,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35958 deadline: 1731857946843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:06,847 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:06,847 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:06,847 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:38:06,872 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=345 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/C/4ddba6187bc143ca9fcfc71c90f06b44 2024-11-17T15:38:06,877 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/A/e8d27a803ea24454b72758c25b02ef64 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/e8d27a803ea24454b72758c25b02ef64 2024-11-17T15:38:06,881 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/e8d27a803ea24454b72758c25b02ef64, entries=200, sequenceid=345, filesize=14.4 K 2024-11-17T15:38:06,882 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/B/2b109fdc1665474594783adc4a515f56 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/2b109fdc1665474594783adc4a515f56 2024-11-17T15:38:06,886 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/2b109fdc1665474594783adc4a515f56, entries=150, sequenceid=345, filesize=12.0 K 2024-11-17T15:38:06,887 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/C/4ddba6187bc143ca9fcfc71c90f06b44 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/4ddba6187bc143ca9fcfc71c90f06b44 2024-11-17T15:38:06,891 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/4ddba6187bc143ca9fcfc71c90f06b44, entries=150, sequenceid=345, filesize=12.0 K 2024-11-17T15:38:06,892 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for 5b1940a26f6b6e92cc2f11a0e1deef31 in 1256ms, sequenceid=345, compaction requested=true 2024-11-17T15:38:06,892 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5b1940a26f6b6e92cc2f11a0e1deef31: 2024-11-17T15:38:06,892 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5b1940a26f6b6e92cc2f11a0e1deef31:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T15:38:06,892 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T15:38:06,892 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5b1940a26f6b6e92cc2f11a0e1deef31:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T15:38:06,892 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T15:38:06,892 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T15:38:06,892 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5b1940a26f6b6e92cc2f11a0e1deef31:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T15:38:06,892 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-17T15:38:06,892 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T15:38:06,894 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49920 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T15:38:06,894 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1541): 5b1940a26f6b6e92cc2f11a0e1deef31/B is initiating minor compaction (all files) 2024-11-17T15:38:06,894 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5b1940a26f6b6e92cc2f11a0e1deef31/B in TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31. 2024-11-17T15:38:06,894 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/9c4842868a6b45188c986515c379efed, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/7d8d3eba323c4583b0e62cfe15a6d22a, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/4c4b281e03f242c89611ba2d5a1965f6, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/2b109fdc1665474594783adc4a515f56] into tmpdir=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp, totalSize=48.8 K 2024-11-17T15:38:06,894 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54800 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T15:38:06,894 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1541): 5b1940a26f6b6e92cc2f11a0e1deef31/A is initiating minor compaction (all files) 2024-11-17T15:38:06,895 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5b1940a26f6b6e92cc2f11a0e1deef31/A in TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31. 2024-11-17T15:38:06,895 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/48c7cc7e10a34a7db823c17730b818fa, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/f1c2836660494d6684f1a65a8124c808, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/050b11bf94d945ae95ccf95263eefe19, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/e8d27a803ea24454b72758c25b02ef64] into tmpdir=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp, totalSize=53.5 K 2024-11-17T15:38:06,896 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting 9c4842868a6b45188c986515c379efed, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1731857883225 2024-11-17T15:38:06,897 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 48c7cc7e10a34a7db823c17730b818fa, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1731857883225 2024-11-17T15:38:06,897 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting 7d8d3eba323c4583b0e62cfe15a6d22a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=305, earliestPutTs=1731857883993 2024-11-17T15:38:06,898 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting f1c2836660494d6684f1a65a8124c808, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=305, earliestPutTs=1731857883993 2024-11-17T15:38:06,898 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting 4c4b281e03f242c89611ba2d5a1965f6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1731857884081 2024-11-17T15:38:06,898 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 050b11bf94d945ae95ccf95263eefe19, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1731857884081 2024-11-17T15:38:06,898 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting 2b109fdc1665474594783adc4a515f56, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1731857885303 2024-11-17T15:38:06,899 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting e8d27a803ea24454b72758c25b02ef64, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1731857885303 2024-11-17T15:38:06,901 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-17T15:38:06,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31. 2024-11-17T15:38:06,902 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2902): Flushing 5b1940a26f6b6e92cc2f11a0e1deef31 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-17T15:38:06,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/A/02dfcbd07f5c41edae9f3cfe216e6295 is 50, key is test_row_0/A:col10/1731857885689/Put/seqid=0 2024-11-17T15:38:06,929 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5b1940a26f6b6e92cc2f11a0e1deef31#B#compaction#273 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T15:38:06,929 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/B/79a1dd285ea94e7e90362d9ddb69c23b is 50, key is test_row_0/B:col10/1731857885303/Put/seqid=0 2024-11-17T15:38:06,939 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5b1940a26f6b6e92cc2f11a0e1deef31#A#compaction#274 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T15:38:06,940 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/A/258b9d6300fb42c1a0943bb45a69a1b0 is 50, key is test_row_0/A:col10/1731857885303/Put/seqid=0 2024-11-17T15:38:06,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742147_1323 (size=12301) 2024-11-17T15:38:06,947 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=362 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/A/02dfcbd07f5c41edae9f3cfe216e6295 2024-11-17T15:38:06,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742148_1324 (size=13153) 2024-11-17T15:38:06,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/B/a604fb63620b41f8905341d7aa5f7702 is 50, key is test_row_0/B:col10/1731857885689/Put/seqid=0 2024-11-17T15:38:06,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742149_1325 (size=13153) 2024-11-17T15:38:06,995 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/A/258b9d6300fb42c1a0943bb45a69a1b0 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/258b9d6300fb42c1a0943bb45a69a1b0 2024-11-17T15:38:06,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742150_1326 (size=12301) 2024-11-17T15:38:06,997 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=362 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/B/a604fb63620b41f8905341d7aa5f7702 2024-11-17T15:38:07,002 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5b1940a26f6b6e92cc2f11a0e1deef31/A of 5b1940a26f6b6e92cc2f11a0e1deef31 into 258b9d6300fb42c1a0943bb45a69a1b0(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T15:38:07,002 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5b1940a26f6b6e92cc2f11a0e1deef31: 2024-11-17T15:38:07,002 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., storeName=5b1940a26f6b6e92cc2f11a0e1deef31/A, priority=12, startTime=1731857886892; duration=0sec 2024-11-17T15:38:07,002 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T15:38:07,002 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5b1940a26f6b6e92cc2f11a0e1deef31:A 2024-11-17T15:38:07,002 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T15:38:07,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/C/aa75dc8b66af4f1f8506c69a66b4fcdb is 50, key is test_row_0/C:col10/1731857885689/Put/seqid=0 2024-11-17T15:38:07,009 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49920 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T15:38:07,009 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1541): 5b1940a26f6b6e92cc2f11a0e1deef31/C is initiating minor compaction (all files) 2024-11-17T15:38:07,009 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5b1940a26f6b6e92cc2f11a0e1deef31/C in TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31. 2024-11-17T15:38:07,009 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/eca894e019e4423f9b31f52367577268, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/d4b48d87873f4f7a8a383622b5c5822e, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/12c3f61c3ee1488c9b0aeb6fb923268b, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/4ddba6187bc143ca9fcfc71c90f06b44] into tmpdir=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp, totalSize=48.8 K 2024-11-17T15:38:07,009 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting eca894e019e4423f9b31f52367577268, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1731857883225 2024-11-17T15:38:07,010 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting d4b48d87873f4f7a8a383622b5c5822e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=305, earliestPutTs=1731857883993 2024-11-17T15:38:07,010 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 12c3f61c3ee1488c9b0aeb6fb923268b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1731857884081 2024-11-17T15:38:07,011 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4ddba6187bc143ca9fcfc71c90f06b44, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1731857885303 2024-11-17T15:38:07,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742151_1327 (size=12301) 2024-11-17T15:38:07,021 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5b1940a26f6b6e92cc2f11a0e1deef31#C#compaction#277 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T15:38:07,023 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/C/24fdb9acf86a426eb096fec57db26429 is 50, key is test_row_0/C:col10/1731857885303/Put/seqid=0 2024-11-17T15:38:07,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742152_1328 (size=13153) 2024-11-17T15:38:07,367 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/B/79a1dd285ea94e7e90362d9ddb69c23b as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/79a1dd285ea94e7e90362d9ddb69c23b 2024-11-17T15:38:07,372 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5b1940a26f6b6e92cc2f11a0e1deef31/B of 5b1940a26f6b6e92cc2f11a0e1deef31 into 79a1dd285ea94e7e90362d9ddb69c23b(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T15:38:07,372 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5b1940a26f6b6e92cc2f11a0e1deef31: 2024-11-17T15:38:07,372 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., storeName=5b1940a26f6b6e92cc2f11a0e1deef31/B, priority=12, startTime=1731857886892; duration=0sec 2024-11-17T15:38:07,373 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T15:38:07,373 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5b1940a26f6b6e92cc2f11a0e1deef31:B 2024-11-17T15:38:07,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(8855): Flush requested on 5b1940a26f6b6e92cc2f11a0e1deef31 2024-11-17T15:38:07,404 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31. as already flushing 2024-11-17T15:38:07,415 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=362 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/C/aa75dc8b66af4f1f8506c69a66b4fcdb 2024-11-17T15:38:07,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/A/02dfcbd07f5c41edae9f3cfe216e6295 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/02dfcbd07f5c41edae9f3cfe216e6295 2024-11-17T15:38:07,425 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/02dfcbd07f5c41edae9f3cfe216e6295, entries=150, sequenceid=362, filesize=12.0 K 2024-11-17T15:38:07,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/B/a604fb63620b41f8905341d7aa5f7702 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/a604fb63620b41f8905341d7aa5f7702 2024-11-17T15:38:07,429 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/a604fb63620b41f8905341d7aa5f7702, entries=150, sequenceid=362, filesize=12.0 K 2024-11-17T15:38:07,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/C/aa75dc8b66af4f1f8506c69a66b4fcdb as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/aa75dc8b66af4f1f8506c69a66b4fcdb 2024-11-17T15:38:07,436 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/C/24fdb9acf86a426eb096fec57db26429 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/24fdb9acf86a426eb096fec57db26429 2024-11-17T15:38:07,437 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/aa75dc8b66af4f1f8506c69a66b4fcdb, entries=150, sequenceid=362, filesize=12.0 K 2024-11-17T15:38:07,438 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=73.80 KB/75570 for 5b1940a26f6b6e92cc2f11a0e1deef31 in 536ms, sequenceid=362, compaction requested=false 2024-11-17T15:38:07,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2603): Flush status journal for 5b1940a26f6b6e92cc2f11a0e1deef31: 2024-11-17T15:38:07,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31. 2024-11-17T15:38:07,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=85 2024-11-17T15:38:07,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.HMaster(4169): Remote procedure done, pid=85 2024-11-17T15:38:07,438 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T15:38:07,439 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5b1940a26f6b6e92cc2f11a0e1deef31 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-17T15:38:07,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(8855): Flush requested on 5b1940a26f6b6e92cc2f11a0e1deef31 2024-11-17T15:38:07,443 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=85, resume processing ppid=84 2024-11-17T15:38:07,443 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, ppid=84, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7640 sec 2024-11-17T15:38:07,443 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5b1940a26f6b6e92cc2f11a0e1deef31/C of 5b1940a26f6b6e92cc2f11a0e1deef31 into 24fdb9acf86a426eb096fec57db26429(size=12.8 K), total size for store is 24.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T15:38:07,443 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5b1940a26f6b6e92cc2f11a0e1deef31: 2024-11-17T15:38:07,443 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., storeName=5b1940a26f6b6e92cc2f11a0e1deef31/C, priority=12, startTime=1731857886892; duration=0sec 2024-11-17T15:38:07,443 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T15:38:07,443 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5b1940a26f6b6e92cc2f11a0e1deef31:C 2024-11-17T15:38:07,445 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees in 1.7700 sec 2024-11-17T15:38:07,445 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/A/b82a69410e7d42e5990dcc738ff3a648 is 50, key is test_row_0/A:col10/1731857887435/Put/seqid=0 2024-11-17T15:38:07,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742153_1329 (size=12301) 2024-11-17T15:38:07,512 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:07,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35954 deadline: 1731857947506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:07,513 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:07,513 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:07,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35884 deadline: 1731857947507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:07,513 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:07,513 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:38:07,513 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:07,513 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:07,513 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:38:07,517 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:07,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35940 deadline: 1731857947513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:07,518 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:07,518 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:07,518 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:38:07,625 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:07,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35954 deadline: 1731857947623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:07,625 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:07,625 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:07,625 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:07,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35884 deadline: 1731857947623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:07,625 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:38:07,626 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:07,626 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:07,626 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:38:07,626 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:07,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35940 deadline: 1731857947623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:07,626 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:07,626 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:07,626 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:38:07,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-17T15:38:07,833 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T15:38:07,833 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T15:38:07,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] procedure2.ProcedureExecutor(1139): Stored pid=86, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees 2024-11-17T15:38:07,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-17T15:38:07,835 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=86, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T15:38:07,836 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:07,836 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=86, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T15:38:07,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35954 deadline: 1731857947833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:07,836 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=87, ppid=86, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T15:38:07,836 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:07,837 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:07,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35884 deadline: 1731857947833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:07,837 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:07,837 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:38:07,837 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:07,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35940 deadline: 1731857947833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:07,837 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:07,837 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:07,837 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:38:07,837 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:07,837 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:07,837 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:38:07,851 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=379 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/A/b82a69410e7d42e5990dcc738ff3a648 2024-11-17T15:38:07,856 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:07,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35958 deadline: 1731857947853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:07,856 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 7a780d55532c,32955,1731857800648 for 'test_row_1' in 5b1940a26f6b6e92cc2f11a0e1deef31 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2156 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T15:38:07,857 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:07,857 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:07,857 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:38:07,859 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/B/46f2348474844b71b4970fbec77c7924 is 50, key is test_row_0/B:col10/1731857887435/Put/seqid=0 2024-11-17T15:38:07,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742154_1330 (size=12301) 2024-11-17T15:38:07,866 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=379 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/B/46f2348474844b71b4970fbec77c7924 2024-11-17T15:38:07,873 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/C/f1ca288eca7846919af0c950bf3f9d24 is 50, key is test_row_0/C:col10/1731857887435/Put/seqid=0 2024-11-17T15:38:07,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742155_1331 (size=12301) 2024-11-17T15:38:07,891 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=379 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/C/f1ca288eca7846919af0c950bf3f9d24 2024-11-17T15:38:07,896 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/A/b82a69410e7d42e5990dcc738ff3a648 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/b82a69410e7d42e5990dcc738ff3a648 2024-11-17T15:38:07,900 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/b82a69410e7d42e5990dcc738ff3a648, entries=150, sequenceid=379, filesize=12.0 K 2024-11-17T15:38:07,901 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/B/46f2348474844b71b4970fbec77c7924 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/46f2348474844b71b4970fbec77c7924 2024-11-17T15:38:07,905 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/46f2348474844b71b4970fbec77c7924, entries=150, sequenceid=379, filesize=12.0 K 2024-11-17T15:38:07,907 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/C/f1ca288eca7846919af0c950bf3f9d24 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/f1ca288eca7846919af0c950bf3f9d24 2024-11-17T15:38:07,910 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/f1ca288eca7846919af0c950bf3f9d24, entries=150, sequenceid=379, filesize=12.0 K 2024-11-17T15:38:07,911 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 5b1940a26f6b6e92cc2f11a0e1deef31 in 472ms, sequenceid=379, compaction requested=true 2024-11-17T15:38:07,911 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5b1940a26f6b6e92cc2f11a0e1deef31: 2024-11-17T15:38:07,911 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5b1940a26f6b6e92cc2f11a0e1deef31:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T15:38:07,911 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T15:38:07,911 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T15:38:07,912 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T15:38:07,912 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5b1940a26f6b6e92cc2f11a0e1deef31:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T15:38:07,912 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T15:38:07,912 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5b1940a26f6b6e92cc2f11a0e1deef31:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T15:38:07,912 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T15:38:07,913 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T15:38:07,913 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T15:38:07,913 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1541): 5b1940a26f6b6e92cc2f11a0e1deef31/B is initiating minor compaction (all files) 2024-11-17T15:38:07,913 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1541): 5b1940a26f6b6e92cc2f11a0e1deef31/A is initiating minor compaction (all files) 2024-11-17T15:38:07,913 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5b1940a26f6b6e92cc2f11a0e1deef31/B in TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31. 2024-11-17T15:38:07,913 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5b1940a26f6b6e92cc2f11a0e1deef31/A in TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31. 2024-11-17T15:38:07,913 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/79a1dd285ea94e7e90362d9ddb69c23b, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/a604fb63620b41f8905341d7aa5f7702, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/46f2348474844b71b4970fbec77c7924] into tmpdir=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp, totalSize=36.9 K 2024-11-17T15:38:07,913 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/258b9d6300fb42c1a0943bb45a69a1b0, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/02dfcbd07f5c41edae9f3cfe216e6295, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/b82a69410e7d42e5990dcc738ff3a648] into tmpdir=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp, totalSize=36.9 K 2024-11-17T15:38:07,915 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting 79a1dd285ea94e7e90362d9ddb69c23b, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1731857885303 2024-11-17T15:38:07,915 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 258b9d6300fb42c1a0943bb45a69a1b0, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1731857885303 2024-11-17T15:38:07,915 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting a604fb63620b41f8905341d7aa5f7702, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=362, earliestPutTs=1731857885689 2024-11-17T15:38:07,915 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 02dfcbd07f5c41edae9f3cfe216e6295, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=362, earliestPutTs=1731857885689 2024-11-17T15:38:07,916 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting b82a69410e7d42e5990dcc738ff3a648, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=379, earliestPutTs=1731857887424 2024-11-17T15:38:07,916 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting 46f2348474844b71b4970fbec77c7924, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=379, earliestPutTs=1731857887424 2024-11-17T15:38:07,925 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5b1940a26f6b6e92cc2f11a0e1deef31#B#compaction#281 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T15:38:07,925 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5b1940a26f6b6e92cc2f11a0e1deef31#A#compaction#282 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T15:38:07,926 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/A/02b277d943ab4b9bbb20f44488a16624 is 50, key is test_row_0/A:col10/1731857887435/Put/seqid=0 2024-11-17T15:38:07,926 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/B/4925a4975b724942af43fe1cad83976b is 50, key is test_row_0/B:col10/1731857887435/Put/seqid=0 2024-11-17T15:38:07,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742157_1333 (size=13255) 2024-11-17T15:38:07,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742156_1332 (size=13255) 2024-11-17T15:38:07,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-17T15:38:07,949 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/A/02b277d943ab4b9bbb20f44488a16624 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/02b277d943ab4b9bbb20f44488a16624 2024-11-17T15:38:07,954 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5b1940a26f6b6e92cc2f11a0e1deef31/A of 5b1940a26f6b6e92cc2f11a0e1deef31 into 02b277d943ab4b9bbb20f44488a16624(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T15:38:07,954 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5b1940a26f6b6e92cc2f11a0e1deef31: 2024-11-17T15:38:07,954 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., storeName=5b1940a26f6b6e92cc2f11a0e1deef31/A, priority=13, startTime=1731857887911; duration=0sec 2024-11-17T15:38:07,954 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T15:38:07,954 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5b1940a26f6b6e92cc2f11a0e1deef31:A 2024-11-17T15:38:07,954 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T15:38:07,956 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T15:38:07,956 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1541): 5b1940a26f6b6e92cc2f11a0e1deef31/C is initiating minor compaction (all files) 2024-11-17T15:38:07,957 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5b1940a26f6b6e92cc2f11a0e1deef31/C in TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31. 2024-11-17T15:38:07,957 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/24fdb9acf86a426eb096fec57db26429, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/aa75dc8b66af4f1f8506c69a66b4fcdb, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/f1ca288eca7846919af0c950bf3f9d24] into tmpdir=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp, totalSize=36.9 K 2024-11-17T15:38:07,957 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 24fdb9acf86a426eb096fec57db26429, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1731857885303 2024-11-17T15:38:07,957 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting aa75dc8b66af4f1f8506c69a66b4fcdb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=362, earliestPutTs=1731857885689 2024-11-17T15:38:07,958 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting f1ca288eca7846919af0c950bf3f9d24, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=379, earliestPutTs=1731857887424 2024-11-17T15:38:07,965 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5b1940a26f6b6e92cc2f11a0e1deef31#C#compaction#283 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T15:38:07,965 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/C/a97e39b229e24ba0ae67a848f40903c5 is 50, key is test_row_0/C:col10/1731857887435/Put/seqid=0 2024-11-17T15:38:07,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742158_1334 (size=13255) 2024-11-17T15:38:07,975 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/C/a97e39b229e24ba0ae67a848f40903c5 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/a97e39b229e24ba0ae67a848f40903c5 2024-11-17T15:38:07,980 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5b1940a26f6b6e92cc2f11a0e1deef31/C of 5b1940a26f6b6e92cc2f11a0e1deef31 into a97e39b229e24ba0ae67a848f40903c5(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T15:38:07,980 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5b1940a26f6b6e92cc2f11a0e1deef31: 2024-11-17T15:38:07,980 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., storeName=5b1940a26f6b6e92cc2f11a0e1deef31/C, priority=13, startTime=1731857887912; duration=0sec 2024-11-17T15:38:07,980 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T15:38:07,980 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5b1940a26f6b6e92cc2f11a0e1deef31:C 2024-11-17T15:38:07,989 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-17T15:38:07,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31. 2024-11-17T15:38:07,989 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2902): Flushing 5b1940a26f6b6e92cc2f11a0e1deef31 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-17T15:38:07,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/A/06a3600c2a15444aad960a8638765007 is 50, key is test_row_0/A:col10/1731857887506/Put/seqid=0 2024-11-17T15:38:07,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742159_1335 (size=12301) 2024-11-17T15:38:08,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(8855): Flush requested on 5b1940a26f6b6e92cc2f11a0e1deef31 2024-11-17T15:38:08,146 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31. as already flushing 2024-11-17T15:38:08,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-17T15:38:08,164 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:08,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35954 deadline: 1731857948159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:08,165 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:08,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35884 deadline: 1731857948161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:08,165 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:08,165 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:08,165 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:38:08,165 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:08,165 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:08,165 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:08,165 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:38:08,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35940 deadline: 1731857948162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:08,168 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:08,169 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:08,169 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:38:08,274 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:08,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35954 deadline: 1731857948272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:08,275 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:08,275 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:08,275 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:38:08,275 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:08,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35884 deadline: 1731857948273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:08,276 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:08,276 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:08,276 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:08,276 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:38:08,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35940 deadline: 1731857948273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:08,276 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:08,276 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:08,276 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:38:08,339 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/B/4925a4975b724942af43fe1cad83976b as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/4925a4975b724942af43fe1cad83976b 2024-11-17T15:38:08,344 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5b1940a26f6b6e92cc2f11a0e1deef31/B of 5b1940a26f6b6e92cc2f11a0e1deef31 into 4925a4975b724942af43fe1cad83976b(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T15:38:08,344 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5b1940a26f6b6e92cc2f11a0e1deef31: 2024-11-17T15:38:08,344 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., storeName=5b1940a26f6b6e92cc2f11a0e1deef31/B, priority=13, startTime=1731857887911; duration=0sec 2024-11-17T15:38:08,344 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T15:38:08,344 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5b1940a26f6b6e92cc2f11a0e1deef31:B 2024-11-17T15:38:08,398 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=403 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/A/06a3600c2a15444aad960a8638765007 2024-11-17T15:38:08,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/B/6e698a7abfae4f89a464436bf5c3b83a is 50, key is test_row_0/B:col10/1731857887506/Put/seqid=0 2024-11-17T15:38:08,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742160_1336 (size=12301) 2024-11-17T15:38:08,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-17T15:38:08,486 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:08,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35954 deadline: 1731857948483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:08,486 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:08,486 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:08,486 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:08,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35884 deadline: 1731857948483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:08,486 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:08,486 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:38:08,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35940 deadline: 1731857948483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:08,487 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:08,487 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:08,487 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:08,487 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:38:08,487 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:08,487 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:38:08,795 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:08,795 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:08,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35954 deadline: 1731857948793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:08,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35884 deadline: 1731857948793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:08,796 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:08,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35940 deadline: 1731857948793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:08,796 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:08,796 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:08,796 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:08,796 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:08,796 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:08,796 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:08,796 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:38:08,796 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:38:08,796 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:38:08,811 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=403 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/B/6e698a7abfae4f89a464436bf5c3b83a 2024-11-17T15:38:08,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/C/a257909395b341f8bab35ec6a86d8495 is 50, key is test_row_0/C:col10/1731857887506/Put/seqid=0 2024-11-17T15:38:08,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742161_1337 (size=12301) 2024-11-17T15:38:08,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-17T15:38:09,091 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-17T15:38:09,223 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=403 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/C/a257909395b341f8bab35ec6a86d8495 2024-11-17T15:38:09,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/A/06a3600c2a15444aad960a8638765007 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/06a3600c2a15444aad960a8638765007 2024-11-17T15:38:09,232 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/06a3600c2a15444aad960a8638765007, entries=150, sequenceid=403, filesize=12.0 K 2024-11-17T15:38:09,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/B/6e698a7abfae4f89a464436bf5c3b83a as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/6e698a7abfae4f89a464436bf5c3b83a 2024-11-17T15:38:09,237 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/6e698a7abfae4f89a464436bf5c3b83a, entries=150, sequenceid=403, filesize=12.0 K 2024-11-17T15:38:09,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/C/a257909395b341f8bab35ec6a86d8495 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/a257909395b341f8bab35ec6a86d8495 2024-11-17T15:38:09,243 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/a257909395b341f8bab35ec6a86d8495, entries=150, sequenceid=403, filesize=12.0 K 2024-11-17T15:38:09,243 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for 5b1940a26f6b6e92cc2f11a0e1deef31 in 1254ms, sequenceid=403, compaction requested=false 2024-11-17T15:38:09,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2603): Flush status journal for 5b1940a26f6b6e92cc2f11a0e1deef31: 2024-11-17T15:38:09,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31. 2024-11-17T15:38:09,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=87 2024-11-17T15:38:09,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.HMaster(4169): Remote procedure done, pid=87 2024-11-17T15:38:09,247 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=87, resume processing ppid=86 2024-11-17T15:38:09,247 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, ppid=86, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4090 sec 2024-11-17T15:38:09,249 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees in 1.4150 sec 2024-11-17T15:38:09,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(8855): Flush requested on 5b1940a26f6b6e92cc2f11a0e1deef31 2024-11-17T15:38:09,305 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T15:38:09,305 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5b1940a26f6b6e92cc2f11a0e1deef31 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-17T15:38:09,310 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/A/286f14b3720c4b65abb48721fd56c0db is 50, key is test_row_0/A:col10/1731857889303/Put/seqid=0 2024-11-17T15:38:09,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742162_1338 (size=12301) 2024-11-17T15:38:09,336 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:09,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35884 deadline: 1731857949332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:09,337 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:09,337 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:09,337 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:38:09,339 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:09,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35954 deadline: 1731857949335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:09,339 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:09,339 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:09,339 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:09,339 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:38:09,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35940 deadline: 1731857949336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:09,340 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:09,340 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:09,340 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:38:09,447 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:09,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35884 deadline: 1731857949443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:09,447 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:09,447 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:09,447 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:09,447 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:38:09,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35954 deadline: 1731857949443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:09,448 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:09,448 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:09,447 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:09,448 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:38:09,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35940 deadline: 1731857949443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:09,448 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:09,448 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:09,448 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:38:09,656 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:09,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35884 deadline: 1731857949653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:09,656 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:09,656 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:09,656 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:09,656 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:38:09,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35954 deadline: 1731857949653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:09,656 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:09,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35940 deadline: 1731857949653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:09,657 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:09,657 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:09,657 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:38:09,657 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:09,657 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:09,657 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:38:09,714 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=420 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/A/286f14b3720c4b65abb48721fd56c0db 2024-11-17T15:38:09,727 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/B/0fbf96cf251740acb10a6188fd3220fb is 50, key is test_row_0/B:col10/1731857889303/Put/seqid=0 2024-11-17T15:38:09,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742163_1339 (size=12301) 2024-11-17T15:38:09,864 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:09,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35958 deadline: 1731857949863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:09,865 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 7a780d55532c,32955,1731857800648 for 'test_row_1' in 5b1940a26f6b6e92cc2f11a0e1deef31 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4164 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T15:38:09,865 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:09,865 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:09,866 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:38:09,963 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:09,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35884 deadline: 1731857949963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:09,964 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:09,964 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:09,964 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:38:09,966 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:09,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35954 deadline: 1731857949963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:09,967 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:09,967 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:09,967 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:09,967 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:38:09,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35940 deadline: 1731857949963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:09,967 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:09,967 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:09,967 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:38:09,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-17T15:38:09,983 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T15:38:09,984 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T15:38:09,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] procedure2.ProcedureExecutor(1139): Stored pid=88, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=88, table=TestAcidGuarantees 2024-11-17T15:38:09,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-17T15:38:09,985 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=88, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=88, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T15:38:09,986 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=88, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=88, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T15:38:09,986 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=89, ppid=88, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T15:38:10,075 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:10,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35902 deadline: 1731857950073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:10,075 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 7a780d55532c,32955,1731857800648 for 'test_row_2' in 5b1940a26f6b6e92cc2f11a0e1deef31 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18304 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T15:38:10,076 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:10,076 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:10,076 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:38:10,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-17T15:38:10,139 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=420 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/B/0fbf96cf251740acb10a6188fd3220fb 2024-11-17T15:38:10,139 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=89 2024-11-17T15:38:10,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31. 2024-11-17T15:38:10,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31. as already flushing 2024-11-17T15:38:10,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31. 2024-11-17T15:38:10,140 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] handler.RSProcedureHandler(58): pid=89 java.io.IOException: Unable to complete flush {ENCODED => 5b1940a26f6b6e92cc2f11a0e1deef31, NAME => 'TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:10,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=89 java.io.IOException: Unable to complete flush {ENCODED => 5b1940a26f6b6e92cc2f11a0e1deef31, NAME => 'TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:10,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=89 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b1940a26f6b6e92cc2f11a0e1deef31, NAME => 'TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b1940a26f6b6e92cc2f11a0e1deef31, NAME => 'TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:10,147 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/C/f21ec659b16749bc86d3c817f5cdb182 is 50, key is test_row_0/C:col10/1731857889303/Put/seqid=0 2024-11-17T15:38:10,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742164_1340 (size=12301) 2024-11-17T15:38:10,151 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=420 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/C/f21ec659b16749bc86d3c817f5cdb182 2024-11-17T15:38:10,159 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/A/286f14b3720c4b65abb48721fd56c0db as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/286f14b3720c4b65abb48721fd56c0db 2024-11-17T15:38:10,163 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/286f14b3720c4b65abb48721fd56c0db, entries=150, sequenceid=420, filesize=12.0 K 2024-11-17T15:38:10,164 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/B/0fbf96cf251740acb10a6188fd3220fb as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/0fbf96cf251740acb10a6188fd3220fb 2024-11-17T15:38:10,168 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/0fbf96cf251740acb10a6188fd3220fb, entries=150, sequenceid=420, filesize=12.0 K 2024-11-17T15:38:10,169 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/C/f21ec659b16749bc86d3c817f5cdb182 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/f21ec659b16749bc86d3c817f5cdb182 2024-11-17T15:38:10,173 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/f21ec659b16749bc86d3c817f5cdb182, entries=150, sequenceid=420, filesize=12.0 K 2024-11-17T15:38:10,173 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for 5b1940a26f6b6e92cc2f11a0e1deef31 in 868ms, sequenceid=420, compaction requested=true 2024-11-17T15:38:10,173 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5b1940a26f6b6e92cc2f11a0e1deef31: 2024-11-17T15:38:10,174 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5b1940a26f6b6e92cc2f11a0e1deef31:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T15:38:10,174 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T15:38:10,174 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5b1940a26f6b6e92cc2f11a0e1deef31:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T15:38:10,174 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T15:38:10,174 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5b1940a26f6b6e92cc2f11a0e1deef31:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T15:38:10,174 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T15:38:10,174 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T15:38:10,174 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T15:38:10,174 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T15:38:10,174 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T15:38:10,175 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1541): 5b1940a26f6b6e92cc2f11a0e1deef31/A is initiating minor compaction (all files) 2024-11-17T15:38:10,175 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1541): 5b1940a26f6b6e92cc2f11a0e1deef31/B is initiating minor compaction (all files) 2024-11-17T15:38:10,175 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5b1940a26f6b6e92cc2f11a0e1deef31/A in TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31. 2024-11-17T15:38:10,175 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5b1940a26f6b6e92cc2f11a0e1deef31/B in TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31. 2024-11-17T15:38:10,175 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/02b277d943ab4b9bbb20f44488a16624, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/06a3600c2a15444aad960a8638765007, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/286f14b3720c4b65abb48721fd56c0db] into tmpdir=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp, totalSize=37.0 K 2024-11-17T15:38:10,175 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/4925a4975b724942af43fe1cad83976b, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/6e698a7abfae4f89a464436bf5c3b83a, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/0fbf96cf251740acb10a6188fd3220fb] into tmpdir=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp, totalSize=37.0 K 2024-11-17T15:38:10,175 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 02b277d943ab4b9bbb20f44488a16624, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=379, earliestPutTs=1731857887424 2024-11-17T15:38:10,175 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting 4925a4975b724942af43fe1cad83976b, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=379, earliestPutTs=1731857887424 2024-11-17T15:38:10,176 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 06a3600c2a15444aad960a8638765007, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=403, earliestPutTs=1731857887505 2024-11-17T15:38:10,176 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting 6e698a7abfae4f89a464436bf5c3b83a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=403, earliestPutTs=1731857887505 2024-11-17T15:38:10,176 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting 0fbf96cf251740acb10a6188fd3220fb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=420, earliestPutTs=1731857888158 2024-11-17T15:38:10,176 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 286f14b3720c4b65abb48721fd56c0db, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=420, earliestPutTs=1731857888158 2024-11-17T15:38:10,184 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5b1940a26f6b6e92cc2f11a0e1deef31#B#compaction#290 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T15:38:10,185 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/B/62ed75327dc3447a8865730a3d6d8eca is 50, key is test_row_0/B:col10/1731857889303/Put/seqid=0 2024-11-17T15:38:10,186 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5b1940a26f6b6e92cc2f11a0e1deef31#A#compaction#291 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T15:38:10,187 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/A/db88daa320e14cba807dc2806f9e627a is 50, key is test_row_0/A:col10/1731857889303/Put/seqid=0 2024-11-17T15:38:10,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742165_1341 (size=13357) 2024-11-17T15:38:10,194 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/B/62ed75327dc3447a8865730a3d6d8eca as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/62ed75327dc3447a8865730a3d6d8eca 2024-11-17T15:38:10,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742166_1342 (size=13357) 2024-11-17T15:38:10,201 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5b1940a26f6b6e92cc2f11a0e1deef31/B of 5b1940a26f6b6e92cc2f11a0e1deef31 into 62ed75327dc3447a8865730a3d6d8eca(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T15:38:10,201 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5b1940a26f6b6e92cc2f11a0e1deef31: 2024-11-17T15:38:10,201 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., storeName=5b1940a26f6b6e92cc2f11a0e1deef31/B, priority=13, startTime=1731857890174; duration=0sec 2024-11-17T15:38:10,201 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T15:38:10,202 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5b1940a26f6b6e92cc2f11a0e1deef31:B 2024-11-17T15:38:10,202 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T15:38:10,203 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/A/db88daa320e14cba807dc2806f9e627a as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/db88daa320e14cba807dc2806f9e627a 2024-11-17T15:38:10,204 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T15:38:10,204 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1541): 5b1940a26f6b6e92cc2f11a0e1deef31/C is initiating minor compaction (all files) 2024-11-17T15:38:10,204 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5b1940a26f6b6e92cc2f11a0e1deef31/C in TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31. 2024-11-17T15:38:10,204 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/a97e39b229e24ba0ae67a848f40903c5, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/a257909395b341f8bab35ec6a86d8495, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/f21ec659b16749bc86d3c817f5cdb182] into tmpdir=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp, totalSize=37.0 K 2024-11-17T15:38:10,205 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting a97e39b229e24ba0ae67a848f40903c5, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=379, earliestPutTs=1731857887424 2024-11-17T15:38:10,205 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting a257909395b341f8bab35ec6a86d8495, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=403, earliestPutTs=1731857887505 2024-11-17T15:38:10,206 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting f21ec659b16749bc86d3c817f5cdb182, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=420, earliestPutTs=1731857888158 2024-11-17T15:38:10,208 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5b1940a26f6b6e92cc2f11a0e1deef31/A of 5b1940a26f6b6e92cc2f11a0e1deef31 into db88daa320e14cba807dc2806f9e627a(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T15:38:10,208 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5b1940a26f6b6e92cc2f11a0e1deef31: 2024-11-17T15:38:10,208 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., storeName=5b1940a26f6b6e92cc2f11a0e1deef31/A, priority=13, startTime=1731857890173; duration=0sec 2024-11-17T15:38:10,208 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T15:38:10,208 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5b1940a26f6b6e92cc2f11a0e1deef31:A 2024-11-17T15:38:10,213 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5b1940a26f6b6e92cc2f11a0e1deef31#C#compaction#292 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T15:38:10,213 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/C/35879cca6df04da5bd721ebc7827e91b is 50, key is test_row_0/C:col10/1731857889303/Put/seqid=0 2024-11-17T15:38:10,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742167_1343 (size=13357) 2024-11-17T15:38:10,224 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/C/35879cca6df04da5bd721ebc7827e91b as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/35879cca6df04da5bd721ebc7827e91b 2024-11-17T15:38:10,230 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5b1940a26f6b6e92cc2f11a0e1deef31/C of 5b1940a26f6b6e92cc2f11a0e1deef31 into 35879cca6df04da5bd721ebc7827e91b(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T15:38:10,230 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5b1940a26f6b6e92cc2f11a0e1deef31: 2024-11-17T15:38:10,230 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., storeName=5b1940a26f6b6e92cc2f11a0e1deef31/C, priority=13, startTime=1731857890174; duration=0sec 2024-11-17T15:38:10,230 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T15:38:10,230 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5b1940a26f6b6e92cc2f11a0e1deef31:C 2024-11-17T15:38:10,293 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=89 2024-11-17T15:38:10,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31. 2024-11-17T15:38:10,293 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegion(2902): Flushing 5b1940a26f6b6e92cc2f11a0e1deef31 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-17T15:38:10,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/A/476606fe8898423bb6f78daa655c3203 is 50, key is test_row_0/A:col10/1731857889333/Put/seqid=0 2024-11-17T15:38:10,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-17T15:38:10,303 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742168_1344 (size=12301) 2024-11-17T15:38:10,304 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=443 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/A/476606fe8898423bb6f78daa655c3203 2024-11-17T15:38:10,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/B/7d2e22ebfe374c3fb2c6000edde151ec is 50, key is test_row_0/B:col10/1731857889333/Put/seqid=0 2024-11-17T15:38:10,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742169_1345 (size=12301) 2024-11-17T15:38:10,317 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=443 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/B/7d2e22ebfe374c3fb2c6000edde151ec 2024-11-17T15:38:10,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/C/ab07c8d5cdda49d28951000bf9b5237a is 50, key is test_row_0/C:col10/1731857889333/Put/seqid=0 2024-11-17T15:38:10,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742170_1346 (size=12301) 2024-11-17T15:38:10,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(8855): Flush requested on 5b1940a26f6b6e92cc2f11a0e1deef31 2024-11-17T15:38:10,476 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31. as already flushing 2024-11-17T15:38:10,477 INFO [Thread-1165 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1165. 2024-11-17T15:38:10,477 INFO [Thread-1148 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1148. 2024-11-17T15:38:10,477 DEBUG [Thread-1165 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T15:38:10,477 DEBUG [Thread-1165 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:38:10,477 DEBUG [Thread-1148 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T15:38:10,477 DEBUG [Thread-1148 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:38:10,477 DEBUG [Thread-1148 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:38:10,477 DEBUG [Thread-1165 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:38:10,477 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T15:38:10,477 INFO [Thread-1146 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1146. 2024-11-17T15:38:10,477 DEBUG [Thread-1146 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T15:38:10,477 DEBUG [Thread-1146 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:38:10,477 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T15:38:10,478 DEBUG [Thread-1146 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:38:10,478 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T15:38:10,480 INFO [Thread-1161 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1161. 2024-11-17T15:38:10,480 DEBUG [Thread-1161 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T15:38:10,480 DEBUG [Thread-1161 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:38:10,480 DEBUG [Thread-1161 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:38:10,480 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T15:38:10,481 INFO [Thread-1159 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1159. 2024-11-17T15:38:10,481 DEBUG [Thread-1159 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T15:38:10,481 DEBUG [Thread-1159 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:38:10,481 DEBUG [Thread-1159 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:38:10,481 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T15:38:10,481 INFO [Thread-1163 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1163. 2024-11-17T15:38:10,481 DEBUG [Thread-1163 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T15:38:10,481 DEBUG [Thread-1163 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:38:10,481 DEBUG [Thread-1163 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:38:10,481 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T15:38:10,481 INFO [Thread-1154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1154. 2024-11-17T15:38:10,481 DEBUG [Thread-1154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T15:38:10,481 DEBUG [Thread-1154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:38:10,482 DEBUG [Thread-1154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:38:10,482 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T15:38:10,482 INFO [Thread-1157 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1157. 2024-11-17T15:38:10,482 DEBUG [Thread-1157 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T15:38:10,482 DEBUG [Thread-1157 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:38:10,482 DEBUG [Thread-1157 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:38:10,482 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T15:38:10,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-17T15:38:10,729 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=443 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/C/ab07c8d5cdda49d28951000bf9b5237a 2024-11-17T15:38:10,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/A/476606fe8898423bb6f78daa655c3203 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/476606fe8898423bb6f78daa655c3203 2024-11-17T15:38:10,737 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/476606fe8898423bb6f78daa655c3203, entries=150, sequenceid=443, filesize=12.0 K 2024-11-17T15:38:10,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/B/7d2e22ebfe374c3fb2c6000edde151ec as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/7d2e22ebfe374c3fb2c6000edde151ec 2024-11-17T15:38:10,740 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/7d2e22ebfe374c3fb2c6000edde151ec, entries=150, sequenceid=443, filesize=12.0 K 2024-11-17T15:38:10,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/C/ab07c8d5cdda49d28951000bf9b5237a as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/ab07c8d5cdda49d28951000bf9b5237a 2024-11-17T15:38:10,744 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/ab07c8d5cdda49d28951000bf9b5237a, entries=150, sequenceid=443, filesize=12.0 K 2024-11-17T15:38:10,745 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=26.84 KB/27480 for 5b1940a26f6b6e92cc2f11a0e1deef31 in 452ms, sequenceid=443, compaction requested=false 2024-11-17T15:38:10,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegion(2603): Flush status journal for 5b1940a26f6b6e92cc2f11a0e1deef31: 2024-11-17T15:38:10,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31. 2024-11-17T15:38:10,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=89 2024-11-17T15:38:10,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.HMaster(4169): Remote procedure done, pid=89 2024-11-17T15:38:10,748 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=89, resume processing ppid=88 2024-11-17T15:38:10,748 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, ppid=88, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 760 msec 2024-11-17T15:38:10,749 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=88, table=TestAcidGuarantees in 764 msec 2024-11-17T15:38:11,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-17T15:38:11,124 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T15:38:13,895 INFO [Thread-1152 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1152. 2024-11-17T15:38:13,895 DEBUG [Thread-1152 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T15:38:13,895 DEBUG [Thread-1152 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:38:13,895 DEBUG [Thread-1152 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:38:13,895 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T15:38:20,145 INFO [Thread-1150 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1150. 2024-11-17T15:38:20,145 DEBUG [Thread-1150 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T15:38:20,145 DEBUG [Thread-1150 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:38:20,145 DEBUG [Thread-1150 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:38:20,145 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T15:38:20,145 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-17T15:38:20,145 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 70 2024-11-17T15:38:20,145 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 70 2024-11-17T15:38:20,145 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 14 2024-11-17T15:38:20,145 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 125 2024-11-17T15:38:20,145 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 70 2024-11-17T15:38:20,145 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-17T15:38:20,145 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3369 2024-11-17T15:38:20,145 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3228 2024-11-17T15:38:20,145 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3233 2024-11-17T15:38:20,145 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3346 2024-11-17T15:38:20,145 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3265 2024-11-17T15:38:20,145 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-17T15:38:20,145 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-17T15:38:20,145 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-17T15:38:20,145 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:38:20,146 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:38:20,146 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T15:38:20,146 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-17T15:38:20,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] procedure2.ProcedureExecutor(1139): Stored pid=90, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-17T15:38:20,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-17T15:38:20,149 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731857900149"}]},"ts":"1731857900149"} 2024-11-17T15:38:20,151 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-17T15:38:20,151 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-17T15:38:20,151 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=91, ppid=90, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-17T15:38:20,153 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5b1940a26f6b6e92cc2f11a0e1deef31, UNASSIGN}] 2024-11-17T15:38:20,153 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5b1940a26f6b6e92cc2f11a0e1deef31, UNASSIGN 2024-11-17T15:38:20,154 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=5b1940a26f6b6e92cc2f11a0e1deef31, regionState=CLOSING, regionLocation=7a780d55532c,32955,1731857800648 2024-11-17T15:38:20,155 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5b1940a26f6b6e92cc2f11a0e1deef31, UNASSIGN because future has completed 2024-11-17T15:38:20,156 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-17T15:38:20,156 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=93, ppid=92, state=RUNNABLE, hasLock=false; CloseRegionProcedure 5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648}] 2024-11-17T15:38:20,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-17T15:38:20,308 INFO [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] handler.UnassignRegionHandler(122): Close 5b1940a26f6b6e92cc2f11a0e1deef31 2024-11-17T15:38:20,308 DEBUG [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-17T15:38:20,308 DEBUG [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1722): Closing 5b1940a26f6b6e92cc2f11a0e1deef31, disabling compactions & flushes 2024-11-17T15:38:20,308 INFO [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31. 2024-11-17T15:38:20,308 DEBUG [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31. 2024-11-17T15:38:20,308 DEBUG [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31. after waiting 0 ms 2024-11-17T15:38:20,308 DEBUG [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31. 2024-11-17T15:38:20,309 INFO [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(2902): Flushing 5b1940a26f6b6e92cc2f11a0e1deef31 3/3 column families, dataSize=40.25 KB heapSize=106.22 KB 2024-11-17T15:38:20,312 DEBUG [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/A/514b7c0c823c4452872c9540c2ea88d7 is 50, key is test_row_1/A:col10/1731857893893/Put/seqid=0 2024-11-17T15:38:20,316 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742171_1347 (size=9857) 2024-11-17T15:38:20,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-17T15:38:20,716 INFO [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=452 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/A/514b7c0c823c4452872c9540c2ea88d7 2024-11-17T15:38:20,723 DEBUG [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/B/e2ea28619f634b1bb04f68cbbb789b75 is 50, key is test_row_1/B:col10/1731857893893/Put/seqid=0 2024-11-17T15:38:20,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742172_1348 (size=9857) 2024-11-17T15:38:20,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-17T15:38:21,127 INFO [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=452 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/B/e2ea28619f634b1bb04f68cbbb789b75 2024-11-17T15:38:21,134 DEBUG [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/C/ce3d0c4bde5b4f03bd29b25f4d7dc381 is 50, key is test_row_1/C:col10/1731857893893/Put/seqid=0 2024-11-17T15:38:21,137 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742173_1349 (size=9857) 2024-11-17T15:38:21,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-17T15:38:21,537 INFO [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=452 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/C/ce3d0c4bde5b4f03bd29b25f4d7dc381 2024-11-17T15:38:21,542 DEBUG [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/A/514b7c0c823c4452872c9540c2ea88d7 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/514b7c0c823c4452872c9540c2ea88d7 2024-11-17T15:38:21,545 INFO [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/514b7c0c823c4452872c9540c2ea88d7, entries=100, sequenceid=452, filesize=9.6 K 2024-11-17T15:38:21,546 DEBUG [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/B/e2ea28619f634b1bb04f68cbbb789b75 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/e2ea28619f634b1bb04f68cbbb789b75 2024-11-17T15:38:21,549 INFO [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/e2ea28619f634b1bb04f68cbbb789b75, entries=100, sequenceid=452, filesize=9.6 K 2024-11-17T15:38:21,550 DEBUG [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/.tmp/C/ce3d0c4bde5b4f03bd29b25f4d7dc381 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/ce3d0c4bde5b4f03bd29b25f4d7dc381 2024-11-17T15:38:21,553 INFO [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/ce3d0c4bde5b4f03bd29b25f4d7dc381, entries=100, sequenceid=452, filesize=9.6 K 2024-11-17T15:38:21,554 INFO [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.17 KB/108720, currentSize=0 B/0 for 5b1940a26f6b6e92cc2f11a0e1deef31 in 1246ms, sequenceid=452, compaction requested=true 2024-11-17T15:38:21,554 DEBUG [StoreCloser-TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/9268311b18344df1966a86d170925706, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/8579fd292b254d56a1198dfb452ccb7d, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/934518f8976a4194ad4f338b0602ffd0, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/e175d4579b5e40a5923b52311b44665a, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/000b01d9ad3c462e943c54bd46627c66, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/f8a8c55e77784da49c8ec9c7466fcde2, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/56b92318691f46c195c13dc1ab1b47ba, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/00678687ce4a480a8b5b349009361971, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/bf9d7f64f3704372ac4ecd868f72d203, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/801889d366804c3d8a0f5aa752bf9586, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/a6d1a27e618b4a14a89c3f93aa504041, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/25ccb88d14b443b7bd5bb47fde721ec3, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/01cdd4e6e2d74f2db15ac9f548579b33, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/87210e380159459f92ca022dafac0037, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/2f5e6813c4ee477e9b64609b25bb4ec9, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/207de946c0ba451aaacd41426e69f06b, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/99278c6b91204675b6862a60f044e29f, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/fb73ad213044429bb361c49b5f5b05a4, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/ae878d7290814d9482c7d16a82fcafba, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/0ecda31f47b549b9b77afb68e7c6a551, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/48c7cc7e10a34a7db823c17730b818fa, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/b1f0eb77d5bb4746a099e42ec88e7cc6, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/f1c2836660494d6684f1a65a8124c808, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/050b11bf94d945ae95ccf95263eefe19, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/e8d27a803ea24454b72758c25b02ef64, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/258b9d6300fb42c1a0943bb45a69a1b0, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/02dfcbd07f5c41edae9f3cfe216e6295, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/02b277d943ab4b9bbb20f44488a16624, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/b82a69410e7d42e5990dcc738ff3a648, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/06a3600c2a15444aad960a8638765007, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/286f14b3720c4b65abb48721fd56c0db] to archive 2024-11-17T15:38:21,555 DEBUG [StoreCloser-TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-17T15:38:21,557 DEBUG [StoreCloser-TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/9268311b18344df1966a86d170925706 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/9268311b18344df1966a86d170925706 2024-11-17T15:38:21,558 DEBUG [StoreCloser-TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/8579fd292b254d56a1198dfb452ccb7d to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/8579fd292b254d56a1198dfb452ccb7d 2024-11-17T15:38:21,559 DEBUG [StoreCloser-TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/934518f8976a4194ad4f338b0602ffd0 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/934518f8976a4194ad4f338b0602ffd0 2024-11-17T15:38:21,560 DEBUG [StoreCloser-TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/e175d4579b5e40a5923b52311b44665a to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/e175d4579b5e40a5923b52311b44665a 2024-11-17T15:38:21,561 DEBUG [StoreCloser-TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/000b01d9ad3c462e943c54bd46627c66 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/000b01d9ad3c462e943c54bd46627c66 2024-11-17T15:38:21,562 DEBUG [StoreCloser-TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/f8a8c55e77784da49c8ec9c7466fcde2 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/f8a8c55e77784da49c8ec9c7466fcde2 2024-11-17T15:38:21,563 DEBUG [StoreCloser-TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/56b92318691f46c195c13dc1ab1b47ba to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/56b92318691f46c195c13dc1ab1b47ba 2024-11-17T15:38:21,564 DEBUG [StoreCloser-TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/00678687ce4a480a8b5b349009361971 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/00678687ce4a480a8b5b349009361971 2024-11-17T15:38:21,565 DEBUG [StoreCloser-TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/bf9d7f64f3704372ac4ecd868f72d203 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/bf9d7f64f3704372ac4ecd868f72d203 2024-11-17T15:38:21,566 DEBUG [StoreCloser-TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/801889d366804c3d8a0f5aa752bf9586 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/801889d366804c3d8a0f5aa752bf9586 2024-11-17T15:38:21,567 DEBUG [StoreCloser-TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/a6d1a27e618b4a14a89c3f93aa504041 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/a6d1a27e618b4a14a89c3f93aa504041 2024-11-17T15:38:21,569 DEBUG [StoreCloser-TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/25ccb88d14b443b7bd5bb47fde721ec3 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/25ccb88d14b443b7bd5bb47fde721ec3 2024-11-17T15:38:21,570 DEBUG [StoreCloser-TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/01cdd4e6e2d74f2db15ac9f548579b33 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/01cdd4e6e2d74f2db15ac9f548579b33 2024-11-17T15:38:21,571 DEBUG [StoreCloser-TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/87210e380159459f92ca022dafac0037 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/87210e380159459f92ca022dafac0037 2024-11-17T15:38:21,572 DEBUG [StoreCloser-TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/2f5e6813c4ee477e9b64609b25bb4ec9 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/2f5e6813c4ee477e9b64609b25bb4ec9 2024-11-17T15:38:21,573 DEBUG [StoreCloser-TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/207de946c0ba451aaacd41426e69f06b to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/207de946c0ba451aaacd41426e69f06b 2024-11-17T15:38:21,574 DEBUG [StoreCloser-TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/99278c6b91204675b6862a60f044e29f to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/99278c6b91204675b6862a60f044e29f 2024-11-17T15:38:21,575 DEBUG [StoreCloser-TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/fb73ad213044429bb361c49b5f5b05a4 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/fb73ad213044429bb361c49b5f5b05a4 2024-11-17T15:38:21,577 DEBUG [StoreCloser-TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/ae878d7290814d9482c7d16a82fcafba to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/ae878d7290814d9482c7d16a82fcafba 2024-11-17T15:38:21,578 DEBUG [StoreCloser-TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/0ecda31f47b549b9b77afb68e7c6a551 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/0ecda31f47b549b9b77afb68e7c6a551 2024-11-17T15:38:21,579 DEBUG [StoreCloser-TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/48c7cc7e10a34a7db823c17730b818fa to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/48c7cc7e10a34a7db823c17730b818fa 2024-11-17T15:38:21,580 DEBUG [StoreCloser-TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/b1f0eb77d5bb4746a099e42ec88e7cc6 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/b1f0eb77d5bb4746a099e42ec88e7cc6 2024-11-17T15:38:21,581 DEBUG [StoreCloser-TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/f1c2836660494d6684f1a65a8124c808 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/f1c2836660494d6684f1a65a8124c808 2024-11-17T15:38:21,582 DEBUG [StoreCloser-TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/050b11bf94d945ae95ccf95263eefe19 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/050b11bf94d945ae95ccf95263eefe19 2024-11-17T15:38:21,583 DEBUG [StoreCloser-TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/e8d27a803ea24454b72758c25b02ef64 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/e8d27a803ea24454b72758c25b02ef64 2024-11-17T15:38:21,585 DEBUG [StoreCloser-TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/258b9d6300fb42c1a0943bb45a69a1b0 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/258b9d6300fb42c1a0943bb45a69a1b0 2024-11-17T15:38:21,586 DEBUG [StoreCloser-TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/02dfcbd07f5c41edae9f3cfe216e6295 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/02dfcbd07f5c41edae9f3cfe216e6295 2024-11-17T15:38:21,587 DEBUG [StoreCloser-TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/02b277d943ab4b9bbb20f44488a16624 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/02b277d943ab4b9bbb20f44488a16624 2024-11-17T15:38:21,588 DEBUG [StoreCloser-TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/b82a69410e7d42e5990dcc738ff3a648 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/b82a69410e7d42e5990dcc738ff3a648 2024-11-17T15:38:21,589 DEBUG [StoreCloser-TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/06a3600c2a15444aad960a8638765007 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/06a3600c2a15444aad960a8638765007 2024-11-17T15:38:21,590 DEBUG [StoreCloser-TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/286f14b3720c4b65abb48721fd56c0db to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/286f14b3720c4b65abb48721fd56c0db 2024-11-17T15:38:21,592 DEBUG [StoreCloser-TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/ed7fc068ae084003bff17231c9a342bb, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/6e6f555724274f3fb09c15f9774e92fd, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/8b53743e04b74406b5e549c1aeeb997c, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/dbfad03ca1ba4b49bf880460062387ca, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/c246c40addd44edfa90e375ff06ad5cb, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/3351b845f76e44fba0c5fb014bff1786, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/49cb5d73dbe74f3fbc0551e6f8977fe0, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/e3e2f723b80c45f886bef34eebad2fa6, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/f14a6fec481f4e88a80bd8edf9a28f42, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/e505abe1256f414da8d30356ce7f5ef2, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/e74b14a65a894410861606574c2464f4, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/a3474d1a240542ada73649b5e027640f, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/769f413d271248539065c3d90cdc962b, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/9b94c9f0ab59447ebd9f9da27b06cd54, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/9e8f5e2c82e041af9dec3725f0319242, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/c339b3c0ad10441d893bc2ea050619eb, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/4efdd0e1ddd2439cb59429a3e4c4dbf3, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/71a6304c2d3c474ab274f840d7f60d63, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/e977c8abd3654b8aaa5babe13427bf0d, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/aea528ceb29543a484127fd6b8d2d0b0, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/9c4842868a6b45188c986515c379efed, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/bae8c5325a9d40a5a7658c9b4549082b, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/7d8d3eba323c4583b0e62cfe15a6d22a, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/4c4b281e03f242c89611ba2d5a1965f6, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/79a1dd285ea94e7e90362d9ddb69c23b, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/2b109fdc1665474594783adc4a515f56, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/a604fb63620b41f8905341d7aa5f7702, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/4925a4975b724942af43fe1cad83976b, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/46f2348474844b71b4970fbec77c7924, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/6e698a7abfae4f89a464436bf5c3b83a, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/0fbf96cf251740acb10a6188fd3220fb] to archive 2024-11-17T15:38:21,593 DEBUG [StoreCloser-TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-17T15:38:21,594 DEBUG [StoreCloser-TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/ed7fc068ae084003bff17231c9a342bb to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/ed7fc068ae084003bff17231c9a342bb 2024-11-17T15:38:21,595 DEBUG [StoreCloser-TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/6e6f555724274f3fb09c15f9774e92fd to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/6e6f555724274f3fb09c15f9774e92fd 2024-11-17T15:38:21,596 DEBUG [StoreCloser-TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/8b53743e04b74406b5e549c1aeeb997c to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/8b53743e04b74406b5e549c1aeeb997c 2024-11-17T15:38:21,597 DEBUG [StoreCloser-TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/dbfad03ca1ba4b49bf880460062387ca to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/dbfad03ca1ba4b49bf880460062387ca 2024-11-17T15:38:21,598 DEBUG [StoreCloser-TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/c246c40addd44edfa90e375ff06ad5cb to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/c246c40addd44edfa90e375ff06ad5cb 2024-11-17T15:38:21,599 DEBUG [StoreCloser-TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/3351b845f76e44fba0c5fb014bff1786 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/3351b845f76e44fba0c5fb014bff1786 2024-11-17T15:38:21,600 DEBUG [StoreCloser-TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/49cb5d73dbe74f3fbc0551e6f8977fe0 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/49cb5d73dbe74f3fbc0551e6f8977fe0 2024-11-17T15:38:21,601 DEBUG [StoreCloser-TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/e3e2f723b80c45f886bef34eebad2fa6 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/e3e2f723b80c45f886bef34eebad2fa6 2024-11-17T15:38:21,602 DEBUG [StoreCloser-TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/f14a6fec481f4e88a80bd8edf9a28f42 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/f14a6fec481f4e88a80bd8edf9a28f42 2024-11-17T15:38:21,603 DEBUG [StoreCloser-TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/e505abe1256f414da8d30356ce7f5ef2 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/e505abe1256f414da8d30356ce7f5ef2 2024-11-17T15:38:21,604 DEBUG [StoreCloser-TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/e74b14a65a894410861606574c2464f4 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/e74b14a65a894410861606574c2464f4 2024-11-17T15:38:21,605 DEBUG [StoreCloser-TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/a3474d1a240542ada73649b5e027640f to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/a3474d1a240542ada73649b5e027640f 2024-11-17T15:38:21,606 DEBUG [StoreCloser-TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/769f413d271248539065c3d90cdc962b to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/769f413d271248539065c3d90cdc962b 2024-11-17T15:38:21,607 DEBUG [StoreCloser-TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/9b94c9f0ab59447ebd9f9da27b06cd54 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/9b94c9f0ab59447ebd9f9da27b06cd54 2024-11-17T15:38:21,608 DEBUG [StoreCloser-TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/9e8f5e2c82e041af9dec3725f0319242 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/9e8f5e2c82e041af9dec3725f0319242 2024-11-17T15:38:21,609 DEBUG [StoreCloser-TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/c339b3c0ad10441d893bc2ea050619eb to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/c339b3c0ad10441d893bc2ea050619eb 2024-11-17T15:38:21,609 DEBUG [StoreCloser-TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/4efdd0e1ddd2439cb59429a3e4c4dbf3 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/4efdd0e1ddd2439cb59429a3e4c4dbf3 2024-11-17T15:38:21,610 DEBUG [StoreCloser-TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/71a6304c2d3c474ab274f840d7f60d63 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/71a6304c2d3c474ab274f840d7f60d63 2024-11-17T15:38:21,611 DEBUG [StoreCloser-TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/e977c8abd3654b8aaa5babe13427bf0d to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/e977c8abd3654b8aaa5babe13427bf0d 2024-11-17T15:38:21,612 DEBUG [StoreCloser-TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/aea528ceb29543a484127fd6b8d2d0b0 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/aea528ceb29543a484127fd6b8d2d0b0 2024-11-17T15:38:21,613 DEBUG [StoreCloser-TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/9c4842868a6b45188c986515c379efed to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/9c4842868a6b45188c986515c379efed 2024-11-17T15:38:21,614 DEBUG [StoreCloser-TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/bae8c5325a9d40a5a7658c9b4549082b to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/bae8c5325a9d40a5a7658c9b4549082b 2024-11-17T15:38:21,615 DEBUG [StoreCloser-TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/7d8d3eba323c4583b0e62cfe15a6d22a to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/7d8d3eba323c4583b0e62cfe15a6d22a 2024-11-17T15:38:21,616 DEBUG [StoreCloser-TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/4c4b281e03f242c89611ba2d5a1965f6 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/4c4b281e03f242c89611ba2d5a1965f6 2024-11-17T15:38:21,617 DEBUG [StoreCloser-TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/79a1dd285ea94e7e90362d9ddb69c23b to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/79a1dd285ea94e7e90362d9ddb69c23b 2024-11-17T15:38:21,617 DEBUG [StoreCloser-TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/2b109fdc1665474594783adc4a515f56 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/2b109fdc1665474594783adc4a515f56 2024-11-17T15:38:21,618 DEBUG [StoreCloser-TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/a604fb63620b41f8905341d7aa5f7702 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/a604fb63620b41f8905341d7aa5f7702 2024-11-17T15:38:21,619 DEBUG [StoreCloser-TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/4925a4975b724942af43fe1cad83976b to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/4925a4975b724942af43fe1cad83976b 2024-11-17T15:38:21,620 DEBUG [StoreCloser-TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/46f2348474844b71b4970fbec77c7924 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/46f2348474844b71b4970fbec77c7924 2024-11-17T15:38:21,621 DEBUG [StoreCloser-TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/6e698a7abfae4f89a464436bf5c3b83a to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/6e698a7abfae4f89a464436bf5c3b83a 2024-11-17T15:38:21,622 DEBUG [StoreCloser-TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/0fbf96cf251740acb10a6188fd3220fb to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/0fbf96cf251740acb10a6188fd3220fb 2024-11-17T15:38:21,623 DEBUG [StoreCloser-TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/fc36d5cb8bdc46d798f897b0aeb41488, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/59ade488e18b414ebca0bb91a36e4b43, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/8078048adee444c68dda5ee03376915b, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/fb883a228d4e4bc3be7a2fe3e45b5ed6, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/cd1cb539dca04df3b5a2ffa6341cafff, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/3cc1d8b097034e398ce8ddc63ae36750, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/580315bc583e48da9ceff321fd01b1d4, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/2a9bee46afc2408f95773ca34ea2f3fa, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/ea23e89f185b4331b76ea65be73e5257, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/ec3c907c70ae46feba90245a7a841d44, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/9826019ec99f4819b780bca9e3e9f1d5, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/dc62506d55194b09b619b00225574893, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/676458318fcc4f498acf17fd48d24627, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/83fa1df6dd19428087caf41da89ecffc, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/afbe04b1dcd2492c96a8e74200dceeb5, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/4c35d3bc40a441eb9883282183f70c5f, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/6edfcfa84d5245538fdb2377b787b93d, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/7242f6fc6ec548dba2f2ef0babcfdc47, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/003cdf15f8394f2eb9bf67feb72a80c4, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/87f0885598b2493985ebad87d49e4658, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/eca894e019e4423f9b31f52367577268, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/9178f5d18d664cf583be44f411f675ae, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/d4b48d87873f4f7a8a383622b5c5822e, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/12c3f61c3ee1488c9b0aeb6fb923268b, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/24fdb9acf86a426eb096fec57db26429, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/4ddba6187bc143ca9fcfc71c90f06b44, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/aa75dc8b66af4f1f8506c69a66b4fcdb, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/a97e39b229e24ba0ae67a848f40903c5, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/f1ca288eca7846919af0c950bf3f9d24, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/a257909395b341f8bab35ec6a86d8495, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/f21ec659b16749bc86d3c817f5cdb182] to archive 2024-11-17T15:38:21,624 DEBUG [StoreCloser-TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-17T15:38:21,625 DEBUG [StoreCloser-TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/fc36d5cb8bdc46d798f897b0aeb41488 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/fc36d5cb8bdc46d798f897b0aeb41488 2024-11-17T15:38:21,626 DEBUG [StoreCloser-TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/59ade488e18b414ebca0bb91a36e4b43 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/59ade488e18b414ebca0bb91a36e4b43 2024-11-17T15:38:21,627 DEBUG [StoreCloser-TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/8078048adee444c68dda5ee03376915b to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/8078048adee444c68dda5ee03376915b 2024-11-17T15:38:21,628 DEBUG [StoreCloser-TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/fb883a228d4e4bc3be7a2fe3e45b5ed6 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/fb883a228d4e4bc3be7a2fe3e45b5ed6 2024-11-17T15:38:21,629 DEBUG [StoreCloser-TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/cd1cb539dca04df3b5a2ffa6341cafff to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/cd1cb539dca04df3b5a2ffa6341cafff 2024-11-17T15:38:21,630 DEBUG [StoreCloser-TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/3cc1d8b097034e398ce8ddc63ae36750 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/3cc1d8b097034e398ce8ddc63ae36750 2024-11-17T15:38:21,631 DEBUG [StoreCloser-TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/580315bc583e48da9ceff321fd01b1d4 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/580315bc583e48da9ceff321fd01b1d4 2024-11-17T15:38:21,632 DEBUG [StoreCloser-TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/2a9bee46afc2408f95773ca34ea2f3fa to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/2a9bee46afc2408f95773ca34ea2f3fa 2024-11-17T15:38:21,634 DEBUG [StoreCloser-TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/ea23e89f185b4331b76ea65be73e5257 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/ea23e89f185b4331b76ea65be73e5257 2024-11-17T15:38:21,635 DEBUG [StoreCloser-TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/ec3c907c70ae46feba90245a7a841d44 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/ec3c907c70ae46feba90245a7a841d44 2024-11-17T15:38:21,636 DEBUG [StoreCloser-TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/9826019ec99f4819b780bca9e3e9f1d5 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/9826019ec99f4819b780bca9e3e9f1d5 2024-11-17T15:38:21,637 DEBUG [StoreCloser-TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/dc62506d55194b09b619b00225574893 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/dc62506d55194b09b619b00225574893 2024-11-17T15:38:21,638 DEBUG [StoreCloser-TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/676458318fcc4f498acf17fd48d24627 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/676458318fcc4f498acf17fd48d24627 2024-11-17T15:38:21,639 DEBUG [StoreCloser-TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/83fa1df6dd19428087caf41da89ecffc to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/83fa1df6dd19428087caf41da89ecffc 2024-11-17T15:38:21,639 DEBUG [StoreCloser-TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/afbe04b1dcd2492c96a8e74200dceeb5 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/afbe04b1dcd2492c96a8e74200dceeb5 2024-11-17T15:38:21,641 DEBUG [StoreCloser-TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/4c35d3bc40a441eb9883282183f70c5f to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/4c35d3bc40a441eb9883282183f70c5f 2024-11-17T15:38:21,642 DEBUG [StoreCloser-TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/6edfcfa84d5245538fdb2377b787b93d to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/6edfcfa84d5245538fdb2377b787b93d 2024-11-17T15:38:21,643 DEBUG [StoreCloser-TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/7242f6fc6ec548dba2f2ef0babcfdc47 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/7242f6fc6ec548dba2f2ef0babcfdc47 2024-11-17T15:38:21,643 DEBUG [StoreCloser-TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/003cdf15f8394f2eb9bf67feb72a80c4 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/003cdf15f8394f2eb9bf67feb72a80c4 2024-11-17T15:38:21,644 DEBUG [StoreCloser-TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/87f0885598b2493985ebad87d49e4658 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/87f0885598b2493985ebad87d49e4658 2024-11-17T15:38:21,645 DEBUG [StoreCloser-TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/eca894e019e4423f9b31f52367577268 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/eca894e019e4423f9b31f52367577268 2024-11-17T15:38:21,646 DEBUG [StoreCloser-TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/9178f5d18d664cf583be44f411f675ae to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/9178f5d18d664cf583be44f411f675ae 2024-11-17T15:38:21,647 DEBUG [StoreCloser-TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/d4b48d87873f4f7a8a383622b5c5822e to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/d4b48d87873f4f7a8a383622b5c5822e 2024-11-17T15:38:21,647 DEBUG [StoreCloser-TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/12c3f61c3ee1488c9b0aeb6fb923268b to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/12c3f61c3ee1488c9b0aeb6fb923268b 2024-11-17T15:38:21,648 DEBUG [StoreCloser-TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/24fdb9acf86a426eb096fec57db26429 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/24fdb9acf86a426eb096fec57db26429 2024-11-17T15:38:21,649 DEBUG [StoreCloser-TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/4ddba6187bc143ca9fcfc71c90f06b44 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/4ddba6187bc143ca9fcfc71c90f06b44 2024-11-17T15:38:21,650 DEBUG [StoreCloser-TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/aa75dc8b66af4f1f8506c69a66b4fcdb to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/aa75dc8b66af4f1f8506c69a66b4fcdb 2024-11-17T15:38:21,651 DEBUG [StoreCloser-TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/a97e39b229e24ba0ae67a848f40903c5 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/a97e39b229e24ba0ae67a848f40903c5 2024-11-17T15:38:21,651 DEBUG [StoreCloser-TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/f1ca288eca7846919af0c950bf3f9d24 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/f1ca288eca7846919af0c950bf3f9d24 2024-11-17T15:38:21,652 DEBUG [StoreCloser-TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/a257909395b341f8bab35ec6a86d8495 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/a257909395b341f8bab35ec6a86d8495 2024-11-17T15:38:21,653 DEBUG [StoreCloser-TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/f21ec659b16749bc86d3c817f5cdb182 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/f21ec659b16749bc86d3c817f5cdb182 2024-11-17T15:38:21,659 DEBUG [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/recovered.edits/455.seqid, newMaxSeqId=455, maxSeqId=1 2024-11-17T15:38:21,660 INFO [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31. 2024-11-17T15:38:21,660 DEBUG [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1676): Region close journal for 5b1940a26f6b6e92cc2f11a0e1deef31: Waiting for close lock at 1731857900308Running coprocessor pre-close hooks at 1731857900308Disabling compacts and flushes for region at 1731857900308Disabling writes for close at 1731857900308Obtaining lock to block concurrent updates at 1731857900309 (+1 ms)Preparing flush snapshotting stores in 5b1940a26f6b6e92cc2f11a0e1deef31 at 1731857900309Finished memstore snapshotting TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31., syncing WAL and waiting on mvcc, flushsize=dataSize=41220, getHeapSize=108720, getOffHeapSize=0, getCellsCount=900 at 1731857900309Flushing stores of TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31. at 1731857900309Flushing 5b1940a26f6b6e92cc2f11a0e1deef31/A: creating writer at 1731857900309Flushing 5b1940a26f6b6e92cc2f11a0e1deef31/A: appending metadata at 1731857900312 (+3 ms)Flushing 5b1940a26f6b6e92cc2f11a0e1deef31/A: closing flushed file at 1731857900312Flushing 5b1940a26f6b6e92cc2f11a0e1deef31/B: creating writer at 1731857900720 (+408 ms)Flushing 5b1940a26f6b6e92cc2f11a0e1deef31/B: appending metadata at 1731857900723 (+3 ms)Flushing 5b1940a26f6b6e92cc2f11a0e1deef31/B: closing flushed file at 1731857900723Flushing 5b1940a26f6b6e92cc2f11a0e1deef31/C: creating writer at 1731857901131 (+408 ms)Flushing 5b1940a26f6b6e92cc2f11a0e1deef31/C: appending metadata at 1731857901133 (+2 ms)Flushing 5b1940a26f6b6e92cc2f11a0e1deef31/C: closing flushed file at 1731857901133Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4a3c6c3c: reopening flushed file at 1731857901541 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7180f647: reopening flushed file at 1731857901545 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@23ce70aa: reopening flushed file at 1731857901549 (+4 ms)Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.17 KB/108720, currentSize=0 B/0 for 5b1940a26f6b6e92cc2f11a0e1deef31 in 1246ms, sequenceid=452, compaction requested=true at 1731857901554 (+5 ms)Writing region close event to WAL at 1731857901655 (+101 ms)Running coprocessor post-close hooks at 1731857901659 (+4 ms)Closed at 1731857901659 2024-11-17T15:38:21,662 INFO [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] handler.UnassignRegionHandler(157): Closed 5b1940a26f6b6e92cc2f11a0e1deef31 2024-11-17T15:38:21,662 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=5b1940a26f6b6e92cc2f11a0e1deef31, regionState=CLOSED 2024-11-17T15:38:21,664 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=93, ppid=92, state=RUNNABLE, hasLock=false; CloseRegionProcedure 5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 because future has completed 2024-11-17T15:38:21,667 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=93, resume processing ppid=92 2024-11-17T15:38:21,667 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, ppid=92, state=SUCCESS, hasLock=false; CloseRegionProcedure 5b1940a26f6b6e92cc2f11a0e1deef31, server=7a780d55532c,32955,1731857800648 in 1.5090 sec 2024-11-17T15:38:21,671 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=92, resume processing ppid=91 2024-11-17T15:38:21,671 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, ppid=91, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5b1940a26f6b6e92cc2f11a0e1deef31, UNASSIGN in 1.5140 sec 2024-11-17T15:38:21,674 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=91, resume processing ppid=90 2024-11-17T15:38:21,675 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, ppid=90, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5210 sec 2024-11-17T15:38:21,676 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731857901676"}]},"ts":"1731857901676"} 2024-11-17T15:38:21,678 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-17T15:38:21,678 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-17T15:38:21,680 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5320 sec 2024-11-17T15:38:22,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-17T15:38:22,303 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-17T15:38:22,304 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-17T15:38:22,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] procedure2.ProcedureExecutor(1139): Stored pid=94, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T15:38:22,306 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=94, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T15:38:22,306 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=94, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T15:38:22,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=94 2024-11-17T15:38:22,313 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31 2024-11-17T15:38:22,315 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A, FileablePath, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B, FileablePath, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C, FileablePath, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/recovered.edits] 2024-11-17T15:38:22,318 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/476606fe8898423bb6f78daa655c3203 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/476606fe8898423bb6f78daa655c3203 2024-11-17T15:38:22,319 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/514b7c0c823c4452872c9540c2ea88d7 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/514b7c0c823c4452872c9540c2ea88d7 2024-11-17T15:38:22,320 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/db88daa320e14cba807dc2806f9e627a to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/A/db88daa320e14cba807dc2806f9e627a 2024-11-17T15:38:22,323 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/62ed75327dc3447a8865730a3d6d8eca to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/62ed75327dc3447a8865730a3d6d8eca 2024-11-17T15:38:22,324 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/7d2e22ebfe374c3fb2c6000edde151ec to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/7d2e22ebfe374c3fb2c6000edde151ec 2024-11-17T15:38:22,325 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/e2ea28619f634b1bb04f68cbbb789b75 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/B/e2ea28619f634b1bb04f68cbbb789b75 2024-11-17T15:38:22,327 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/35879cca6df04da5bd721ebc7827e91b to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/35879cca6df04da5bd721ebc7827e91b 2024-11-17T15:38:22,328 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/ab07c8d5cdda49d28951000bf9b5237a to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/ab07c8d5cdda49d28951000bf9b5237a 2024-11-17T15:38:22,329 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/ce3d0c4bde5b4f03bd29b25f4d7dc381 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/C/ce3d0c4bde5b4f03bd29b25f4d7dc381 2024-11-17T15:38:22,332 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/recovered.edits/455.seqid to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31/recovered.edits/455.seqid 2024-11-17T15:38:22,333 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/5b1940a26f6b6e92cc2f11a0e1deef31 2024-11-17T15:38:22,333 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-17T15:38:22,335 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=94, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T15:38:22,338 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-17T15:38:22,340 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-17T15:38:22,341 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=94, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T15:38:22,341 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-17T15:38:22,341 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731857902341"}]},"ts":"9223372036854775807"} 2024-11-17T15:38:22,343 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-17T15:38:22,343 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 5b1940a26f6b6e92cc2f11a0e1deef31, NAME => 'TestAcidGuarantees,,1731857868284.5b1940a26f6b6e92cc2f11a0e1deef31.', STARTKEY => '', ENDKEY => ''}] 2024-11-17T15:38:22,343 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-17T15:38:22,343 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731857902343"}]},"ts":"9223372036854775807"} 2024-11-17T15:38:22,345 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-17T15:38:22,345 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=94, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T15:38:22,346 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 41 msec 2024-11-17T15:38:22,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=94 2024-11-17T15:38:22,414 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-17T15:38:22,414 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-17T15:38:22,423 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testGetAtomicity Thread=237 (was 238), OpenFileDescriptor=445 (was 449), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=385 (was 371) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=3119 (was 3219) 2024-11-17T15:38:22,431 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobScanAtomicity Thread=237, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=385, ProcessCount=11, AvailableMemoryMB=3119 2024-11-17T15:38:22,432 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-17T15:38:22,432 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-17T15:38:22,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] procedure2.ProcedureExecutor(1139): Stored pid=95, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-17T15:38:22,434 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=95, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-17T15:38:22,434 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:22,435 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 95 2024-11-17T15:38:22,435 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=95, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-17T15:38:22,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-17T15:38:22,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742174_1350 (size=959) 2024-11-17T15:38:22,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-17T15:38:22,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-17T15:38:22,842 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => fdca08c93967309fb9c5dcd33e10e7d5, NAME => 'TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206 2024-11-17T15:38:22,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742175_1351 (size=53) 2024-11-17T15:38:23,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-17T15:38:23,247 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T15:38:23,248 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing fdca08c93967309fb9c5dcd33e10e7d5, disabling compactions & flushes 2024-11-17T15:38:23,248 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. 2024-11-17T15:38:23,248 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. 2024-11-17T15:38:23,248 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. after waiting 0 ms 2024-11-17T15:38:23,248 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. 2024-11-17T15:38:23,248 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. 2024-11-17T15:38:23,248 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for fdca08c93967309fb9c5dcd33e10e7d5: Waiting for close lock at 1731857903248Disabling compacts and flushes for region at 1731857903248Disabling writes for close at 1731857903248Writing region close event to WAL at 1731857903248Closed at 1731857903248 2024-11-17T15:38:23,249 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=95, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-17T15:38:23,249 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731857903249"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731857903249"}]},"ts":"1731857903249"} 2024-11-17T15:38:23,251 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-17T15:38:23,252 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=95, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-17T15:38:23,252 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731857903252"}]},"ts":"1731857903252"} 2024-11-17T15:38:23,253 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-17T15:38:23,253 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=95, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fdca08c93967309fb9c5dcd33e10e7d5, ASSIGN}] 2024-11-17T15:38:23,254 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=96, ppid=95, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fdca08c93967309fb9c5dcd33e10e7d5, ASSIGN 2024-11-17T15:38:23,255 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=96, ppid=95, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=fdca08c93967309fb9c5dcd33e10e7d5, ASSIGN; state=OFFLINE, location=7a780d55532c,32955,1731857800648; forceNewPlan=false, retain=false 2024-11-17T15:38:23,406 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=96 updating hbase:meta row=fdca08c93967309fb9c5dcd33e10e7d5, regionState=OPENING, regionLocation=7a780d55532c,32955,1731857800648 2024-11-17T15:38:23,407 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=96, ppid=95, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=fdca08c93967309fb9c5dcd33e10e7d5, ASSIGN because future has completed 2024-11-17T15:38:23,408 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=97, ppid=96, state=RUNNABLE, hasLock=false; OpenRegionProcedure fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648}] 2024-11-17T15:38:23,564 INFO [RS_OPEN_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_REGION, pid=97}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. 2024-11-17T15:38:23,564 DEBUG [RS_OPEN_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_REGION, pid=97}] regionserver.HRegion(7752): Opening region: {ENCODED => fdca08c93967309fb9c5dcd33e10e7d5, NAME => 'TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.', STARTKEY => '', ENDKEY => ''} 2024-11-17T15:38:23,564 DEBUG [RS_OPEN_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_REGION, pid=97}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees fdca08c93967309fb9c5dcd33e10e7d5 2024-11-17T15:38:23,565 DEBUG [RS_OPEN_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_REGION, pid=97}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T15:38:23,565 DEBUG [RS_OPEN_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_REGION, pid=97}] regionserver.HRegion(7794): checking encryption for fdca08c93967309fb9c5dcd33e10e7d5 2024-11-17T15:38:23,565 DEBUG [RS_OPEN_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_REGION, pid=97}] regionserver.HRegion(7797): checking classloading for fdca08c93967309fb9c5dcd33e10e7d5 2024-11-17T15:38:23,566 INFO [StoreOpener-fdca08c93967309fb9c5dcd33e10e7d5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region fdca08c93967309fb9c5dcd33e10e7d5 2024-11-17T15:38:23,567 INFO [StoreOpener-fdca08c93967309fb9c5dcd33e10e7d5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fdca08c93967309fb9c5dcd33e10e7d5 columnFamilyName A 2024-11-17T15:38:23,567 DEBUG [StoreOpener-fdca08c93967309fb9c5dcd33e10e7d5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:23,568 INFO [StoreOpener-fdca08c93967309fb9c5dcd33e10e7d5-1 {}] regionserver.HStore(327): Store=fdca08c93967309fb9c5dcd33e10e7d5/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T15:38:23,568 INFO [StoreOpener-fdca08c93967309fb9c5dcd33e10e7d5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region fdca08c93967309fb9c5dcd33e10e7d5 2024-11-17T15:38:23,570 INFO [StoreOpener-fdca08c93967309fb9c5dcd33e10e7d5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fdca08c93967309fb9c5dcd33e10e7d5 columnFamilyName B 2024-11-17T15:38:23,570 DEBUG [StoreOpener-fdca08c93967309fb9c5dcd33e10e7d5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:23,570 INFO [StoreOpener-fdca08c93967309fb9c5dcd33e10e7d5-1 {}] regionserver.HStore(327): Store=fdca08c93967309fb9c5dcd33e10e7d5/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T15:38:23,570 INFO [StoreOpener-fdca08c93967309fb9c5dcd33e10e7d5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region fdca08c93967309fb9c5dcd33e10e7d5 2024-11-17T15:38:23,571 INFO [StoreOpener-fdca08c93967309fb9c5dcd33e10e7d5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fdca08c93967309fb9c5dcd33e10e7d5 columnFamilyName C 2024-11-17T15:38:23,572 DEBUG [StoreOpener-fdca08c93967309fb9c5dcd33e10e7d5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:23,572 INFO [StoreOpener-fdca08c93967309fb9c5dcd33e10e7d5-1 {}] regionserver.HStore(327): Store=fdca08c93967309fb9c5dcd33e10e7d5/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T15:38:23,572 DEBUG [RS_OPEN_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_REGION, pid=97}] regionserver.HRegion(1038): replaying wal for fdca08c93967309fb9c5dcd33e10e7d5 2024-11-17T15:38:23,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-17T15:38:23,575 DEBUG [RS_OPEN_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_REGION, pid=97}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5 2024-11-17T15:38:23,576 DEBUG [RS_OPEN_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_REGION, pid=97}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5 2024-11-17T15:38:23,577 DEBUG [RS_OPEN_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_REGION, pid=97}] regionserver.HRegion(1048): stopping wal replay for fdca08c93967309fb9c5dcd33e10e7d5 2024-11-17T15:38:23,577 DEBUG [RS_OPEN_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_REGION, pid=97}] regionserver.HRegion(1060): Cleaning up temporary data for fdca08c93967309fb9c5dcd33e10e7d5 2024-11-17T15:38:23,578 DEBUG [RS_OPEN_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_REGION, pid=97}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-17T15:38:23,579 DEBUG [RS_OPEN_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_REGION, pid=97}] regionserver.HRegion(1093): writing seq id for fdca08c93967309fb9c5dcd33e10e7d5 2024-11-17T15:38:23,582 DEBUG [RS_OPEN_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_REGION, pid=97}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-17T15:38:23,582 INFO [RS_OPEN_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_REGION, pid=97}] regionserver.HRegion(1114): Opened fdca08c93967309fb9c5dcd33e10e7d5; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63918064, jitterRate=-0.0475466251373291}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-17T15:38:23,582 DEBUG [RS_OPEN_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_REGION, pid=97}] regionserver.HRegion(1122): Running coprocessor post-open hooks for fdca08c93967309fb9c5dcd33e10e7d5 2024-11-17T15:38:23,583 DEBUG [RS_OPEN_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_REGION, pid=97}] regionserver.HRegion(1006): Region open journal for fdca08c93967309fb9c5dcd33e10e7d5: Running coprocessor pre-open hook at 1731857903565Writing region info on filesystem at 1731857903565Initializing all the Stores at 1731857903565Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731857903565Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731857903566 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731857903566Cleaning up temporary data from old regions at 1731857903577 (+11 ms)Running coprocessor post-open hooks at 1731857903582 (+5 ms)Region opened successfully at 1731857903583 (+1 ms) 2024-11-17T15:38:23,584 INFO [RS_OPEN_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_REGION, pid=97}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., pid=97, masterSystemTime=1731857903560 2024-11-17T15:38:23,586 DEBUG [RS_OPEN_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_REGION, pid=97}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. 2024-11-17T15:38:23,586 INFO [RS_OPEN_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_REGION, pid=97}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. 2024-11-17T15:38:23,586 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=96 updating hbase:meta row=fdca08c93967309fb9c5dcd33e10e7d5, regionState=OPEN, openSeqNum=2, regionLocation=7a780d55532c,32955,1731857800648 2024-11-17T15:38:23,588 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=97, ppid=96, state=RUNNABLE, hasLock=false; OpenRegionProcedure fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 because future has completed 2024-11-17T15:38:23,590 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=97, resume processing ppid=96 2024-11-17T15:38:23,590 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, ppid=96, state=SUCCESS, hasLock=false; OpenRegionProcedure fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 in 181 msec 2024-11-17T15:38:23,593 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=95 2024-11-17T15:38:23,593 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=95, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fdca08c93967309fb9c5dcd33e10e7d5, ASSIGN in 337 msec 2024-11-17T15:38:23,594 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=95, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-17T15:38:23,594 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731857903594"}]},"ts":"1731857903594"} 2024-11-17T15:38:23,595 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-17T15:38:23,596 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=95, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-17T15:38:23,597 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1640 sec 2024-11-17T15:38:24,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-17T15:38:24,583 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-17T15:38:24,584 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7befe57f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:38:24,584 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7a780d55532c,44223,-1 for getting cluster id 2024-11-17T15:38:24,584 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T15:38:24,585 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e01f4cd2-8971-4a73-861a-2ca0deb4f76e' 2024-11-17T15:38:24,585 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T15:38:24,586 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e01f4cd2-8971-4a73-861a-2ca0deb4f76e" 2024-11-17T15:38:24,586 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2121a15, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:38:24,586 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7a780d55532c,44223,-1] 2024-11-17T15:38:24,586 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T15:38:24,586 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:38:24,587 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35652, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T15:38:24,587 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b0b6e4f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:38:24,588 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T15:38:24,589 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7a780d55532c,32955,1731857800648, seqNum=-1] 2024-11-17T15:38:24,589 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T15:38:24,590 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49622, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T15:38:24,590 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-17T15:38:24,591 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 7a780d55532c,44223,1731857799978 2024-11-17T15:38:24,591 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@3b0379b 2024-11-17T15:38:24,591 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-17T15:38:24,592 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35660, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-17T15:38:24,593 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-17T15:38:24,593 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-17T15:38:24,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] procedure2.ProcedureExecutor(1139): Stored pid=98, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-17T15:38:24,603 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742176_1352 (size=995) 2024-11-17T15:38:25,005 DEBUG [PEWorker-1 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-17T15:38:25,005 INFO [PEWorker-1 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-17T15:38:25,007 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=99, ppid=98, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-17T15:38:25,009 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fdca08c93967309fb9c5dcd33e10e7d5, REOPEN/MOVE}] 2024-11-17T15:38:25,010 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=100, ppid=99, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fdca08c93967309fb9c5dcd33e10e7d5, REOPEN/MOVE 2024-11-17T15:38:25,011 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=100 updating hbase:meta row=fdca08c93967309fb9c5dcd33e10e7d5, regionState=CLOSING, regionLocation=7a780d55532c,32955,1731857800648 2024-11-17T15:38:25,012 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=100, ppid=99, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=fdca08c93967309fb9c5dcd33e10e7d5, REOPEN/MOVE because future has completed 2024-11-17T15:38:25,012 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-17T15:38:25,012 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=101, ppid=100, state=RUNNABLE, hasLock=false; CloseRegionProcedure fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648}] 2024-11-17T15:38:25,164 INFO [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=101}] handler.UnassignRegionHandler(122): Close fdca08c93967309fb9c5dcd33e10e7d5 2024-11-17T15:38:25,164 DEBUG [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=101}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-17T15:38:25,164 DEBUG [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=101}] regionserver.HRegion(1722): Closing fdca08c93967309fb9c5dcd33e10e7d5, disabling compactions & flushes 2024-11-17T15:38:25,164 INFO [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=101}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. 2024-11-17T15:38:25,164 DEBUG [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=101}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. 2024-11-17T15:38:25,164 DEBUG [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=101}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. after waiting 0 ms 2024-11-17T15:38:25,164 DEBUG [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=101}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. 2024-11-17T15:38:25,168 DEBUG [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=101}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-17T15:38:25,168 INFO [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=101}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. 2024-11-17T15:38:25,169 DEBUG [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=101}] regionserver.HRegion(1676): Region close journal for fdca08c93967309fb9c5dcd33e10e7d5: Waiting for close lock at 1731857905164Running coprocessor pre-close hooks at 1731857905164Disabling compacts and flushes for region at 1731857905164Disabling writes for close at 1731857905164Writing region close event to WAL at 1731857905165 (+1 ms)Running coprocessor post-close hooks at 1731857905168 (+3 ms)Closed at 1731857905168 2024-11-17T15:38:25,169 WARN [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=101}] regionserver.HRegionServer(3299): Not adding moved region record: fdca08c93967309fb9c5dcd33e10e7d5 to self. 2024-11-17T15:38:25,171 INFO [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=101}] handler.UnassignRegionHandler(157): Closed fdca08c93967309fb9c5dcd33e10e7d5 2024-11-17T15:38:25,171 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=100 updating hbase:meta row=fdca08c93967309fb9c5dcd33e10e7d5, regionState=CLOSED 2024-11-17T15:38:25,172 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=101, ppid=100, state=RUNNABLE, hasLock=false; CloseRegionProcedure fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 because future has completed 2024-11-17T15:38:25,174 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=101, resume processing ppid=100 2024-11-17T15:38:25,175 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, ppid=100, state=SUCCESS, hasLock=false; CloseRegionProcedure fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 in 161 msec 2024-11-17T15:38:25,175 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=100, ppid=99, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=fdca08c93967309fb9c5dcd33e10e7d5, REOPEN/MOVE; state=CLOSED, location=7a780d55532c,32955,1731857800648; forceNewPlan=false, retain=true 2024-11-17T15:38:25,326 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=100 updating hbase:meta row=fdca08c93967309fb9c5dcd33e10e7d5, regionState=OPENING, regionLocation=7a780d55532c,32955,1731857800648 2024-11-17T15:38:25,327 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=100, ppid=99, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=fdca08c93967309fb9c5dcd33e10e7d5, REOPEN/MOVE because future has completed 2024-11-17T15:38:25,327 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=100, state=RUNNABLE, hasLock=false; OpenRegionProcedure fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648}] 2024-11-17T15:38:25,481 INFO [RS_OPEN_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_REGION, pid=102}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. 2024-11-17T15:38:25,481 DEBUG [RS_OPEN_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_REGION, pid=102}] regionserver.HRegion(7752): Opening region: {ENCODED => fdca08c93967309fb9c5dcd33e10e7d5, NAME => 'TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.', STARTKEY => '', ENDKEY => ''} 2024-11-17T15:38:25,481 DEBUG [RS_OPEN_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_REGION, pid=102}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees fdca08c93967309fb9c5dcd33e10e7d5 2024-11-17T15:38:25,481 DEBUG [RS_OPEN_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_REGION, pid=102}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T15:38:25,481 DEBUG [RS_OPEN_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_REGION, pid=102}] regionserver.HRegion(7794): checking encryption for fdca08c93967309fb9c5dcd33e10e7d5 2024-11-17T15:38:25,481 DEBUG [RS_OPEN_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_REGION, pid=102}] regionserver.HRegion(7797): checking classloading for fdca08c93967309fb9c5dcd33e10e7d5 2024-11-17T15:38:25,482 INFO [StoreOpener-fdca08c93967309fb9c5dcd33e10e7d5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region fdca08c93967309fb9c5dcd33e10e7d5 2024-11-17T15:38:25,483 INFO [StoreOpener-fdca08c93967309fb9c5dcd33e10e7d5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fdca08c93967309fb9c5dcd33e10e7d5 columnFamilyName A 2024-11-17T15:38:25,484 DEBUG [StoreOpener-fdca08c93967309fb9c5dcd33e10e7d5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:25,484 INFO [StoreOpener-fdca08c93967309fb9c5dcd33e10e7d5-1 {}] regionserver.HStore(327): Store=fdca08c93967309fb9c5dcd33e10e7d5/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T15:38:25,485 INFO [StoreOpener-fdca08c93967309fb9c5dcd33e10e7d5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region fdca08c93967309fb9c5dcd33e10e7d5 2024-11-17T15:38:25,485 INFO [StoreOpener-fdca08c93967309fb9c5dcd33e10e7d5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fdca08c93967309fb9c5dcd33e10e7d5 columnFamilyName B 2024-11-17T15:38:25,485 DEBUG [StoreOpener-fdca08c93967309fb9c5dcd33e10e7d5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:25,486 INFO [StoreOpener-fdca08c93967309fb9c5dcd33e10e7d5-1 {}] regionserver.HStore(327): Store=fdca08c93967309fb9c5dcd33e10e7d5/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T15:38:25,486 INFO [StoreOpener-fdca08c93967309fb9c5dcd33e10e7d5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region fdca08c93967309fb9c5dcd33e10e7d5 2024-11-17T15:38:25,486 INFO [StoreOpener-fdca08c93967309fb9c5dcd33e10e7d5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fdca08c93967309fb9c5dcd33e10e7d5 columnFamilyName C 2024-11-17T15:38:25,486 DEBUG [StoreOpener-fdca08c93967309fb9c5dcd33e10e7d5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:25,487 INFO [StoreOpener-fdca08c93967309fb9c5dcd33e10e7d5-1 {}] regionserver.HStore(327): Store=fdca08c93967309fb9c5dcd33e10e7d5/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T15:38:25,487 DEBUG [RS_OPEN_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_REGION, pid=102}] regionserver.HRegion(1038): replaying wal for fdca08c93967309fb9c5dcd33e10e7d5 2024-11-17T15:38:25,487 DEBUG [RS_OPEN_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_REGION, pid=102}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5 2024-11-17T15:38:25,488 DEBUG [RS_OPEN_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_REGION, pid=102}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5 2024-11-17T15:38:25,489 DEBUG [RS_OPEN_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_REGION, pid=102}] regionserver.HRegion(1048): stopping wal replay for fdca08c93967309fb9c5dcd33e10e7d5 2024-11-17T15:38:25,489 DEBUG [RS_OPEN_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_REGION, pid=102}] regionserver.HRegion(1060): Cleaning up temporary data for fdca08c93967309fb9c5dcd33e10e7d5 2024-11-17T15:38:25,489 DEBUG [RS_OPEN_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_REGION, pid=102}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-17T15:38:25,490 DEBUG [RS_OPEN_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_REGION, pid=102}] regionserver.HRegion(1093): writing seq id for fdca08c93967309fb9c5dcd33e10e7d5 2024-11-17T15:38:25,491 INFO [RS_OPEN_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_REGION, pid=102}] regionserver.HRegion(1114): Opened fdca08c93967309fb9c5dcd33e10e7d5; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61822649, jitterRate=-0.07877074182033539}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-17T15:38:25,491 DEBUG [RS_OPEN_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_REGION, pid=102}] regionserver.HRegion(1122): Running coprocessor post-open hooks for fdca08c93967309fb9c5dcd33e10e7d5 2024-11-17T15:38:25,491 DEBUG [RS_OPEN_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_REGION, pid=102}] regionserver.HRegion(1006): Region open journal for fdca08c93967309fb9c5dcd33e10e7d5: Running coprocessor pre-open hook at 1731857905481Writing region info on filesystem at 1731857905481Initializing all the Stores at 1731857905482 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731857905482Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731857905482Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731857905482Cleaning up temporary data from old regions at 1731857905489 (+7 ms)Running coprocessor post-open hooks at 1731857905491 (+2 ms)Region opened successfully at 1731857905491 2024-11-17T15:38:25,492 INFO [RS_OPEN_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_REGION, pid=102}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., pid=102, masterSystemTime=1731857905478 2024-11-17T15:38:25,494 DEBUG [RS_OPEN_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_REGION, pid=102}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. 2024-11-17T15:38:25,494 INFO [RS_OPEN_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_REGION, pid=102}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. 2024-11-17T15:38:25,494 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=100 updating hbase:meta row=fdca08c93967309fb9c5dcd33e10e7d5, regionState=OPEN, openSeqNum=5, regionLocation=7a780d55532c,32955,1731857800648 2024-11-17T15:38:25,496 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=102, ppid=100, state=RUNNABLE, hasLock=false; OpenRegionProcedure fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 because future has completed 2024-11-17T15:38:25,498 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=100 2024-11-17T15:38:25,498 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=100, state=SUCCESS, hasLock=false; OpenRegionProcedure fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 in 169 msec 2024-11-17T15:38:25,500 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-11-17T15:38:25,500 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fdca08c93967309fb9c5dcd33e10e7d5, REOPEN/MOVE in 489 msec 2024-11-17T15:38:25,502 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=99, resume processing ppid=98 2024-11-17T15:38:25,502 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, ppid=98, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 493 msec 2024-11-17T15:38:25,504 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 908 msec 2024-11-17T15:38:25,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=98 2024-11-17T15:38:25,504 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-17T15:38:25,505 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b10b9e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:38:25,505 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7a780d55532c,44223,-1 for getting cluster id 2024-11-17T15:38:25,505 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T15:38:25,506 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e01f4cd2-8971-4a73-861a-2ca0deb4f76e' 2024-11-17T15:38:25,506 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T15:38:25,506 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e01f4cd2-8971-4a73-861a-2ca0deb4f76e" 2024-11-17T15:38:25,506 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3aedc725, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:38:25,507 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7a780d55532c,44223,-1] 2024-11-17T15:38:25,507 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T15:38:25,507 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:38:25,507 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35676, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T15:38:25,508 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f916edc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:38:25,508 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21c265f0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:38:25,508 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7a780d55532c,44223,-1 for getting cluster id 2024-11-17T15:38:25,509 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T15:38:25,509 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e01f4cd2-8971-4a73-861a-2ca0deb4f76e' 2024-11-17T15:38:25,509 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T15:38:25,509 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e01f4cd2-8971-4a73-861a-2ca0deb4f76e" 2024-11-17T15:38:25,509 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ff6201d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:38:25,509 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7a780d55532c,44223,-1] 2024-11-17T15:38:25,510 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T15:38:25,510 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:38:25,510 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35694, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T15:38:25,511 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7556edf7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:38:25,511 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c21b87a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:38:25,511 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7a780d55532c,44223,-1 for getting cluster id 2024-11-17T15:38:25,511 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T15:38:25,512 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e01f4cd2-8971-4a73-861a-2ca0deb4f76e' 2024-11-17T15:38:25,512 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T15:38:25,512 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e01f4cd2-8971-4a73-861a-2ca0deb4f76e" 2024-11-17T15:38:25,512 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6acb29fd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:38:25,512 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7a780d55532c,44223,-1] 2024-11-17T15:38:25,512 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T15:38:25,513 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:38:25,513 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35708, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T15:38:25,514 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5bc7116b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:38:25,514 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35e136ad, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:38:25,514 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7a780d55532c,44223,-1 for getting cluster id 2024-11-17T15:38:25,514 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T15:38:25,515 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e01f4cd2-8971-4a73-861a-2ca0deb4f76e' 2024-11-17T15:38:25,515 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T15:38:25,515 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e01f4cd2-8971-4a73-861a-2ca0deb4f76e" 2024-11-17T15:38:25,515 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4620fb6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:38:25,515 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7a780d55532c,44223,-1] 2024-11-17T15:38:25,515 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T15:38:25,515 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:38:25,516 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35718, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T15:38:25,516 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@20ad9e9a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:38:25,517 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f004f7c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:38:25,517 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7a780d55532c,44223,-1 for getting cluster id 2024-11-17T15:38:25,517 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T15:38:25,518 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e01f4cd2-8971-4a73-861a-2ca0deb4f76e' 2024-11-17T15:38:25,518 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T15:38:25,518 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e01f4cd2-8971-4a73-861a-2ca0deb4f76e" 2024-11-17T15:38:25,518 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d77d29f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:38:25,518 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7a780d55532c,44223,-1] 2024-11-17T15:38:25,518 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T15:38:25,518 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:38:25,519 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35744, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T15:38:25,519 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72d3f696, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:38:25,520 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35aa2c8b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:38:25,520 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7a780d55532c,44223,-1 for getting cluster id 2024-11-17T15:38:25,520 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T15:38:25,521 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e01f4cd2-8971-4a73-861a-2ca0deb4f76e' 2024-11-17T15:38:25,521 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T15:38:25,521 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e01f4cd2-8971-4a73-861a-2ca0deb4f76e" 2024-11-17T15:38:25,521 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f415f1d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:38:25,521 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7a780d55532c,44223,-1] 2024-11-17T15:38:25,521 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T15:38:25,521 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:38:25,522 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35762, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T15:38:25,522 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2832c34, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:38:25,523 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53f9edfd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:38:25,523 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7a780d55532c,44223,-1 for getting cluster id 2024-11-17T15:38:25,523 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T15:38:25,523 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e01f4cd2-8971-4a73-861a-2ca0deb4f76e' 2024-11-17T15:38:25,524 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T15:38:25,524 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e01f4cd2-8971-4a73-861a-2ca0deb4f76e" 2024-11-17T15:38:25,524 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f074cb2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:38:25,524 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7a780d55532c,44223,-1] 2024-11-17T15:38:25,524 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T15:38:25,524 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:38:25,525 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35790, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T15:38:25,525 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@18e1a4d3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:38:25,526 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42125f21, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:38:25,526 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7a780d55532c,44223,-1 for getting cluster id 2024-11-17T15:38:25,526 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T15:38:25,526 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e01f4cd2-8971-4a73-861a-2ca0deb4f76e' 2024-11-17T15:38:25,526 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T15:38:25,527 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e01f4cd2-8971-4a73-861a-2ca0deb4f76e" 2024-11-17T15:38:25,527 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5319e118, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:38:25,527 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7a780d55532c,44223,-1] 2024-11-17T15:38:25,527 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T15:38:25,527 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:38:25,528 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35814, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T15:38:25,528 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ff6b197, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:38:25,528 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64b126a0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:38:25,528 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7a780d55532c,44223,-1 for getting cluster id 2024-11-17T15:38:25,529 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T15:38:25,529 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e01f4cd2-8971-4a73-861a-2ca0deb4f76e' 2024-11-17T15:38:25,529 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T15:38:25,529 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e01f4cd2-8971-4a73-861a-2ca0deb4f76e" 2024-11-17T15:38:25,529 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@752e4bd3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:38:25,529 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7a780d55532c,44223,-1] 2024-11-17T15:38:25,530 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T15:38:25,530 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:38:25,530 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35838, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T15:38:25,531 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@18adf899, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:38:25,531 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4fa71035, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:38:25,531 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7a780d55532c,44223,-1 for getting cluster id 2024-11-17T15:38:25,531 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T15:38:25,532 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e01f4cd2-8971-4a73-861a-2ca0deb4f76e' 2024-11-17T15:38:25,532 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T15:38:25,532 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e01f4cd2-8971-4a73-861a-2ca0deb4f76e" 2024-11-17T15:38:25,532 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78a7da74, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:38:25,532 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7a780d55532c,44223,-1] 2024-11-17T15:38:25,532 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T15:38:25,533 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:38:25,533 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35854, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T15:38:25,534 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@711bfcdc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:38:25,537 DEBUG [Thread-1619 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T15:38:25,537 DEBUG [Thread-1617 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T15:38:25,537 DEBUG [Thread-1613 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T15:38:25,537 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7a780d55532c,32955,1731857800648, seqNum=-1] 2024-11-17T15:38:25,537 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7a780d55532c,32955,1731857800648, seqNum=-1] 2024-11-17T15:38:25,537 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T15:38:25,538 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T15:38:25,538 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T15:38:25,538 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7a780d55532c,32955,1731857800648, seqNum=-1] 2024-11-17T15:38:25,538 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T15:38:25,538 DEBUG [Thread-1604 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T15:38:25,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees 2024-11-17T15:38:25,539 DEBUG [Thread-1611 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T15:38:25,539 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49642, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T15:38:25,539 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49626, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T15:38:25,539 DEBUG [Thread-1602 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T15:38:25,540 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49646, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T15:38:25,540 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7a780d55532c,32955,1731857800648, seqNum=-1] 2024-11-17T15:38:25,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-17T15:38:25,540 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T15:38:25,540 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7a780d55532c,32955,1731857800648, seqNum=-1] 2024-11-17T15:38:25,540 DEBUG [Thread-1602 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7a780d55532c,32955,1731857800648, seqNum=-1] 2024-11-17T15:38:25,540 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T15:38:25,540 DEBUG [Thread-1600 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T15:38:25,541 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T15:38:25,541 DEBUG [Thread-1602 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T15:38:25,542 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T15:38:25,542 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49652, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T15:38:25,542 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49654, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T15:38:25,542 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T15:38:25,542 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49668, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T15:38:25,542 DEBUG [Thread-1608 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T15:38:25,542 DEBUG [Thread-1615 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T15:38:25,542 DEBUG [Thread-1606 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T15:38:25,543 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5] 2024-11-17T15:38:25,543 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5] 2024-11-17T15:38:25,545 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5] 2024-11-17T15:38:25,545 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5] 2024-11-17T15:38:25,545 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7a780d55532c,32955,1731857800648, seqNum=-1] 2024-11-17T15:38:25,545 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5] 2024-11-17T15:38:25,545 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T15:38:25,546 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5] 2024-11-17T15:38:25,547 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7a780d55532c,32955,1731857800648, seqNum=-1] 2024-11-17T15:38:25,547 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49680, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T15:38:25,547 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T15:38:25,547 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7a780d55532c,32955,1731857800648, seqNum=-1] 2024-11-17T15:38:25,547 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7a780d55532c,32955,1731857800648, seqNum=-1] 2024-11-17T15:38:25,547 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T15:38:25,547 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T15:38:25,548 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49702, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T15:38:25,548 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49690, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T15:38:25,549 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5] 2024-11-17T15:38:25,550 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5] 2024-11-17T15:38:25,551 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5] 2024-11-17T15:38:25,551 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49700, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T15:38:25,553 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5] 2024-11-17T15:38:25,560 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T15:38:25,560 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fdca08c93967309fb9c5dcd33e10e7d5 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-17T15:38:25,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(8855): Flush requested on fdca08c93967309fb9c5dcd33e10e7d5 2024-11-17T15:38:25,605 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411177d7c0cabdb7249bb87af25dd8dcfbdc2_fdca08c93967309fb9c5dcd33e10e7d5 is 50, key is test_row_0/A:col10/1731857905558/Put/seqid=0 2024-11-17T15:38:25,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742177_1353 (size=17034) 2024-11-17T15:38:25,612 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:25,616 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411177d7c0cabdb7249bb87af25dd8dcfbdc2_fdca08c93967309fb9c5dcd33e10e7d5 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411177d7c0cabdb7249bb87af25dd8dcfbdc2_fdca08c93967309fb9c5dcd33e10e7d5 2024-11-17T15:38:25,620 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:25,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49652 deadline: 1731857965606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:25,620 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/A/a34332a6f86542279c4af1aa6e4e4dfe, store: [table=TestAcidGuarantees family=A region=fdca08c93967309fb9c5dcd33e10e7d5] 2024-11-17T15:38:25,620 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:25,620 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:25,620 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:25,620 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:25,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49700 deadline: 1731857965606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:25,621 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:25,621 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:25,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49680 deadline: 1731857965607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:25,621 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:25,621 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:25,621 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/A/a34332a6f86542279c4af1aa6e4e4dfe is 175, key is test_row_0/A:col10/1731857905558/Put/seqid=0 2024-11-17T15:38:25,621 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:25,621 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:25,621 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:25,630 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:25,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49654 deadline: 1731857965619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:25,630 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:25,630 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:25,630 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:25,632 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:25,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49690 deadline: 1731857965620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:25,633 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:25,633 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:25,633 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:25,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742178_1354 (size=48139) 2024-11-17T15:38:25,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-17T15:38:25,698 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-17T15:38:25,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. 2024-11-17T15:38:25,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. as already flushing 2024-11-17T15:38:25,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. 2024-11-17T15:38:25,699 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => fdca08c93967309fb9c5dcd33e10e7d5, NAME => 'TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:25,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => fdca08c93967309fb9c5dcd33e10e7d5, NAME => 'TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:25,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fdca08c93967309fb9c5dcd33e10e7d5, NAME => 'TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fdca08c93967309fb9c5dcd33e10e7d5, NAME => 'TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:25,731 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:25,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49700 deadline: 1731857965723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:25,731 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:25,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49652 deadline: 1731857965723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:25,732 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:25,732 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:25,732 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:25,732 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:25,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49680 deadline: 1731857965724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:25,732 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:25,732 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:25,732 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:25,733 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:25,733 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:25,733 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:25,743 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:25,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49654 deadline: 1731857965733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:25,744 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:25,744 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:25,744 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:25,747 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:25,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49690 deadline: 1731857965743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:25,748 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:25,748 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:25,748 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:25,852 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-17T15:38:25,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. 2024-11-17T15:38:25,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. as already flushing 2024-11-17T15:38:25,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. 2024-11-17T15:38:25,853 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => fdca08c93967309fb9c5dcd33e10e7d5, NAME => 'TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:25,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => fdca08c93967309fb9c5dcd33e10e7d5, NAME => 'TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:25,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-17T15:38:25,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fdca08c93967309fb9c5dcd33e10e7d5, NAME => 'TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fdca08c93967309fb9c5dcd33e10e7d5, NAME => 'TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:25,951 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:25,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49652 deadline: 1731857965943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:25,952 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:25,952 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:25,952 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:25,952 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:25,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49680 deadline: 1731857965944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:25,952 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:25,952 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:25,952 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:25,952 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:25,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49700 deadline: 1731857965944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:25,953 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:25,953 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:25,953 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:25,957 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:25,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49654 deadline: 1731857965953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:25,958 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:25,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49690 deadline: 1731857965953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:25,958 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:25,958 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:25,958 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:25,958 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:25,958 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:25,958 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:26,006 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-17T15:38:26,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. 2024-11-17T15:38:26,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. as already flushing 2024-11-17T15:38:26,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. 2024-11-17T15:38:26,006 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => fdca08c93967309fb9c5dcd33e10e7d5, NAME => 'TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:26,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => fdca08c93967309fb9c5dcd33e10e7d5, NAME => 'TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:26,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fdca08c93967309fb9c5dcd33e10e7d5, NAME => 'TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fdca08c93967309fb9c5dcd33e10e7d5, NAME => 'TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:26,035 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/A/a34332a6f86542279c4af1aa6e4e4dfe 2024-11-17T15:38:26,061 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/B/64ef4831062f49deb246b5f5662cdd79 is 50, key is test_row_0/B:col10/1731857905558/Put/seqid=0 2024-11-17T15:38:26,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742179_1355 (size=12001) 2024-11-17T15:38:26,160 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-17T15:38:26,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. 2024-11-17T15:38:26,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. as already flushing 2024-11-17T15:38:26,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. 2024-11-17T15:38:26,160 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => fdca08c93967309fb9c5dcd33e10e7d5, NAME => 'TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:26,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => fdca08c93967309fb9c5dcd33e10e7d5, NAME => 'TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:26,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fdca08c93967309fb9c5dcd33e10e7d5, NAME => 'TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fdca08c93967309fb9c5dcd33e10e7d5, NAME => 'TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:26,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-17T15:38:26,270 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:26,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49652 deadline: 1731857966263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:26,271 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:26,271 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:26,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49680 deadline: 1731857966264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:26,271 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:26,271 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:26,271 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:26,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49700 deadline: 1731857966264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:26,271 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:26,271 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:26,271 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:26,271 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:26,271 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:26,271 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:26,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49654 deadline: 1731857966264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:26,271 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:26,272 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:26,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49690 deadline: 1731857966264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:26,272 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:26,272 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:26,272 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:26,272 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:26,272 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:26,272 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:26,313 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-17T15:38:26,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. 2024-11-17T15:38:26,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. as already flushing 2024-11-17T15:38:26,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. 2024-11-17T15:38:26,314 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => fdca08c93967309fb9c5dcd33e10e7d5, NAME => 'TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:26,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => fdca08c93967309fb9c5dcd33e10e7d5, NAME => 'TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:26,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fdca08c93967309fb9c5dcd33e10e7d5, NAME => 'TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fdca08c93967309fb9c5dcd33e10e7d5, NAME => 'TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:26,466 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-17T15:38:26,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. 2024-11-17T15:38:26,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. as already flushing 2024-11-17T15:38:26,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. 2024-11-17T15:38:26,467 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => fdca08c93967309fb9c5dcd33e10e7d5, NAME => 'TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:26,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => fdca08c93967309fb9c5dcd33e10e7d5, NAME => 'TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:26,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fdca08c93967309fb9c5dcd33e10e7d5, NAME => 'TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fdca08c93967309fb9c5dcd33e10e7d5, NAME => 'TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:26,470 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/B/64ef4831062f49deb246b5f5662cdd79 2024-11-17T15:38:26,492 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/C/a261db2ea47f4841b60850d38b4687e9 is 50, key is test_row_0/C:col10/1731857905558/Put/seqid=0 2024-11-17T15:38:26,499 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742180_1356 (size=12001) 2024-11-17T15:38:26,620 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-17T15:38:26,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. 2024-11-17T15:38:26,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. as already flushing 2024-11-17T15:38:26,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. 2024-11-17T15:38:26,620 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => fdca08c93967309fb9c5dcd33e10e7d5, NAME => 'TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:26,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => fdca08c93967309fb9c5dcd33e10e7d5, NAME => 'TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:26,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fdca08c93967309fb9c5dcd33e10e7d5, NAME => 'TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fdca08c93967309fb9c5dcd33e10e7d5, NAME => 'TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:26,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-17T15:38:26,773 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-17T15:38:26,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. 2024-11-17T15:38:26,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. as already flushing 2024-11-17T15:38:26,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. 2024-11-17T15:38:26,773 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => fdca08c93967309fb9c5dcd33e10e7d5, NAME => 'TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:26,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => fdca08c93967309fb9c5dcd33e10e7d5, NAME => 'TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:26,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fdca08c93967309fb9c5dcd33e10e7d5, NAME => 'TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fdca08c93967309fb9c5dcd33e10e7d5, NAME => 'TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:26,788 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:26,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49652 deadline: 1731857966783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:26,789 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:26,789 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:26,789 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:26,789 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:26,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49680 deadline: 1731857966783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:26,789 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:26,789 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:26,789 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:26,789 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:26,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49654 deadline: 1731857966784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:26,790 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:26,790 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:26,790 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:26,790 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:26,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49700 deadline: 1731857966784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:26,790 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:26,790 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:26,790 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:26,790 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:26,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49690 deadline: 1731857966784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:26,790 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:26,790 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:26,790 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:26,901 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/C/a261db2ea47f4841b60850d38b4687e9 2024-11-17T15:38:26,907 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/A/a34332a6f86542279c4af1aa6e4e4dfe as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/A/a34332a6f86542279c4af1aa6e4e4dfe 2024-11-17T15:38:26,912 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/A/a34332a6f86542279c4af1aa6e4e4dfe, entries=250, sequenceid=15, filesize=47.0 K 2024-11-17T15:38:26,913 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/B/64ef4831062f49deb246b5f5662cdd79 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/B/64ef4831062f49deb246b5f5662cdd79 2024-11-17T15:38:26,916 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/B/64ef4831062f49deb246b5f5662cdd79, entries=150, sequenceid=15, filesize=11.7 K 2024-11-17T15:38:26,917 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/C/a261db2ea47f4841b60850d38b4687e9 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/C/a261db2ea47f4841b60850d38b4687e9 2024-11-17T15:38:26,924 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/C/a261db2ea47f4841b60850d38b4687e9, entries=150, sequenceid=15, filesize=11.7 K 2024-11-17T15:38:26,925 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for fdca08c93967309fb9c5dcd33e10e7d5 in 1365ms, sequenceid=15, compaction requested=false 2024-11-17T15:38:26,925 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fdca08c93967309fb9c5dcd33e10e7d5: 2024-11-17T15:38:26,926 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-17T15:38:26,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. 2024-11-17T15:38:26,926 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2902): Flushing fdca08c93967309fb9c5dcd33e10e7d5 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-17T15:38:26,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411174773921fc99941b6b7feafa434ee3809_fdca08c93967309fb9c5dcd33e10e7d5 is 50, key is test_row_0/A:col10/1731857905605/Put/seqid=0 2024-11-17T15:38:26,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742181_1357 (size=12154) 2024-11-17T15:38:27,186 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-17T15:38:27,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:27,358 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411174773921fc99941b6b7feafa434ee3809_fdca08c93967309fb9c5dcd33e10e7d5 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411174773921fc99941b6b7feafa434ee3809_fdca08c93967309fb9c5dcd33e10e7d5 2024-11-17T15:38:27,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/A/1e8ab31ceecf4f959dfc79016730963e, store: [table=TestAcidGuarantees family=A region=fdca08c93967309fb9c5dcd33e10e7d5] 2024-11-17T15:38:27,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/A/1e8ab31ceecf4f959dfc79016730963e is 175, key is test_row_0/A:col10/1731857905605/Put/seqid=0 2024-11-17T15:38:27,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742182_1358 (size=30955) 2024-11-17T15:38:27,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-17T15:38:27,771 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=40, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/A/1e8ab31ceecf4f959dfc79016730963e 2024-11-17T15:38:27,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/B/2256f63d3a7b47dda6900cd2783f893a is 50, key is test_row_0/B:col10/1731857905605/Put/seqid=0 2024-11-17T15:38:27,784 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742183_1359 (size=12001) 2024-11-17T15:38:27,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(8855): Flush requested on fdca08c93967309fb9c5dcd33e10e7d5 2024-11-17T15:38:27,795 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. as already flushing 2024-11-17T15:38:27,810 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:27,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49680 deadline: 1731857967806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:27,811 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:27,811 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:27,811 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:27,815 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:27,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49654 deadline: 1731857967807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:27,815 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:27,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49700 deadline: 1731857967808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:27,816 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:27,816 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:27,816 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:27,818 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:27,818 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:27,818 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:27,822 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:27,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49652 deadline: 1731857967809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:27,822 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:27,822 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:27,822 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:27,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49690 deadline: 1731857967810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:27,822 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:27,823 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:27,823 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:27,823 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:27,921 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:27,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49680 deadline: 1731857967914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:27,921 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:27,921 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:27,921 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:27,931 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:27,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49654 deadline: 1731857967923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:27,931 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:27,932 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:27,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49700 deadline: 1731857967924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:27,932 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:27,932 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:27,932 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:27,932 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:27,932 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:27,937 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:27,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49652 deadline: 1731857967933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:27,938 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:27,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49690 deadline: 1731857967933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:27,938 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:27,938 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:27,938 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:27,939 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:27,939 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:27,939 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:28,141 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:28,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49680 deadline: 1731857968133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:28,143 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:28,143 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:28,143 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:28,150 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:28,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49700 deadline: 1731857968143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:28,151 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:28,151 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:28,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49654 deadline: 1731857968143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:28,151 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:28,151 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:28,151 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:28,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49690 deadline: 1731857968143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:28,151 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:28,151 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:28,151 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:28,152 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:28,151 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:28,152 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:28,152 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:28,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49652 deadline: 1731857968144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:28,152 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:28,152 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:28,152 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:28,187 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/B/2256f63d3a7b47dda6900cd2783f893a 2024-11-17T15:38:28,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/C/46f8fdbd721040fc9fe4f70f89753664 is 50, key is test_row_0/C:col10/1731857905605/Put/seqid=0 2024-11-17T15:38:28,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742184_1360 (size=12001) 2024-11-17T15:38:28,199 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/C/46f8fdbd721040fc9fe4f70f89753664 2024-11-17T15:38:28,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/A/1e8ab31ceecf4f959dfc79016730963e as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/A/1e8ab31ceecf4f959dfc79016730963e 2024-11-17T15:38:28,208 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/A/1e8ab31ceecf4f959dfc79016730963e, entries=150, sequenceid=40, filesize=30.2 K 2024-11-17T15:38:28,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/B/2256f63d3a7b47dda6900cd2783f893a as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/B/2256f63d3a7b47dda6900cd2783f893a 2024-11-17T15:38:28,213 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/B/2256f63d3a7b47dda6900cd2783f893a, entries=150, sequenceid=40, filesize=11.7 K 2024-11-17T15:38:28,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/C/46f8fdbd721040fc9fe4f70f89753664 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/C/46f8fdbd721040fc9fe4f70f89753664 2024-11-17T15:38:28,223 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/C/46f8fdbd721040fc9fe4f70f89753664, entries=150, sequenceid=40, filesize=11.7 K 2024-11-17T15:38:28,224 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for fdca08c93967309fb9c5dcd33e10e7d5 in 1297ms, sequenceid=40, compaction requested=false 2024-11-17T15:38:28,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2603): Flush status journal for fdca08c93967309fb9c5dcd33e10e7d5: 2024-11-17T15:38:28,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. 2024-11-17T15:38:28,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=104 2024-11-17T15:38:28,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.HMaster(4169): Remote procedure done, pid=104 2024-11-17T15:38:28,229 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-11-17T15:38:28,229 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6850 sec 2024-11-17T15:38:28,233 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees in 2.6920 sec 2024-11-17T15:38:28,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(8855): Flush requested on fdca08c93967309fb9c5dcd33e10e7d5 2024-11-17T15:38:28,458 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T15:38:28,458 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fdca08c93967309fb9c5dcd33e10e7d5 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-17T15:38:28,467 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117be95fd6526c34c14b26380d12649db94_fdca08c93967309fb9c5dcd33e10e7d5 is 50, key is test_row_0/A:col10/1731857908458/Put/seqid=0 2024-11-17T15:38:28,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742185_1361 (size=17034) 2024-11-17T15:38:28,518 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:28,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49654 deadline: 1731857968504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:28,518 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:28,518 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:28,518 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:28,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49700 deadline: 1731857968512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:28,518 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:28,519 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:28,519 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:28,519 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:28,524 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:28,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49690 deadline: 1731857968513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:28,525 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:28,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49680 deadline: 1731857968517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:28,525 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:28,525 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:28,525 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:28,525 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:28,526 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:28,526 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:28,531 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:28,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49652 deadline: 1731857968518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:28,531 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:28,531 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:28,531 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:28,633 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:28,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49654 deadline: 1731857968623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:28,634 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:28,634 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:28,634 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:28,634 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:28,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49700 deadline: 1731857968623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:28,634 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:28,634 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:28,634 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:28,642 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:28,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49680 deadline: 1731857968633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:28,642 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:28,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49690 deadline: 1731857968633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:28,643 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:28,643 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:28,643 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:28,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49652 deadline: 1731857968633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:28,643 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:28,643 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:28,643 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:28,643 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:28,644 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:28,644 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:28,644 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:28,850 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:28,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49654 deadline: 1731857968843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:28,851 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:28,851 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:28,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49700 deadline: 1731857968843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:28,851 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:28,851 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:28,851 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:28,851 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:28,851 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:28,863 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:28,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49680 deadline: 1731857968853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:28,863 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:28,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49690 deadline: 1731857968853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:28,864 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:28,864 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:28,864 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:28,864 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:28,864 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:28,864 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:28,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49652 deadline: 1731857968853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:28,864 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:28,864 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:28,864 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:28,864 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:28,875 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:28,879 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117be95fd6526c34c14b26380d12649db94_fdca08c93967309fb9c5dcd33e10e7d5 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117be95fd6526c34c14b26380d12649db94_fdca08c93967309fb9c5dcd33e10e7d5 2024-11-17T15:38:28,879 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/A/c9029bd4d1a448febeb28f67eadc5494, store: [table=TestAcidGuarantees family=A region=fdca08c93967309fb9c5dcd33e10e7d5] 2024-11-17T15:38:28,880 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/A/c9029bd4d1a448febeb28f67eadc5494 is 175, key is test_row_0/A:col10/1731857908458/Put/seqid=0 2024-11-17T15:38:28,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742186_1362 (size=48139) 2024-11-17T15:38:29,171 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:29,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49654 deadline: 1731857969163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:29,171 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:29,171 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:29,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49700 deadline: 1731857969163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:29,171 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:29,171 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:29,172 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:29,172 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:29,172 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:29,183 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:29,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49680 deadline: 1731857969173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:29,183 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:29,183 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:29,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49690 deadline: 1731857969173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:29,183 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:29,184 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:29,184 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:29,184 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:29,184 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:29,184 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:29,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49652 deadline: 1731857969173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:29,184 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:29,184 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:29,184 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:29,287 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=53, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/A/c9029bd4d1a448febeb28f67eadc5494 2024-11-17T15:38:29,294 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/B/173682b39e1d4aa5b74b9bd7c9b83cc1 is 50, key is test_row_0/B:col10/1731857908458/Put/seqid=0 2024-11-17T15:38:29,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742187_1363 (size=12001) 2024-11-17T15:38:29,691 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:29,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49654 deadline: 1731857969683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:29,692 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:29,692 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:29,692 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:29,692 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:29,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49700 deadline: 1731857969683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:29,692 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:29,692 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:29,692 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:29,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-17T15:38:29,693 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T15:38:29,693 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T15:38:29,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees 2024-11-17T15:38:29,695 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T15:38:29,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-17T15:38:29,695 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T15:38:29,695 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T15:38:29,698 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/B/173682b39e1d4aa5b74b9bd7c9b83cc1 2024-11-17T15:38:29,702 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:29,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49680 deadline: 1731857969693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:29,703 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:29,703 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:29,703 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:29,703 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:29,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49690 deadline: 1731857969693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:29,703 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:29,703 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:29,703 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:29,703 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:29,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49652 deadline: 1731857969693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:29,704 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:29,704 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:29,704 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:29,705 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/C/9fe9b74d804c47d3b483e9fb7b172bf3 is 50, key is test_row_0/C:col10/1731857908458/Put/seqid=0 2024-11-17T15:38:29,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742188_1364 (size=12001) 2024-11-17T15:38:29,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-17T15:38:29,848 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-17T15:38:29,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. 2024-11-17T15:38:29,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. as already flushing 2024-11-17T15:38:29,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. 2024-11-17T15:38:29,848 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => fdca08c93967309fb9c5dcd33e10e7d5, NAME => 'TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:29,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => fdca08c93967309fb9c5dcd33e10e7d5, NAME => 'TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:29,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fdca08c93967309fb9c5dcd33e10e7d5, NAME => 'TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fdca08c93967309fb9c5dcd33e10e7d5, NAME => 'TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:30,001 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-17T15:38:30,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. 2024-11-17T15:38:30,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. as already flushing 2024-11-17T15:38:30,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. 2024-11-17T15:38:30,001 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => fdca08c93967309fb9c5dcd33e10e7d5, NAME => 'TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:30,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => fdca08c93967309fb9c5dcd33e10e7d5, NAME => 'TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:30,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fdca08c93967309fb9c5dcd33e10e7d5, NAME => 'TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fdca08c93967309fb9c5dcd33e10e7d5, NAME => 'TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:30,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-17T15:38:30,110 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/C/9fe9b74d804c47d3b483e9fb7b172bf3 2024-11-17T15:38:30,114 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/A/c9029bd4d1a448febeb28f67eadc5494 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/A/c9029bd4d1a448febeb28f67eadc5494 2024-11-17T15:38:30,117 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/A/c9029bd4d1a448febeb28f67eadc5494, entries=250, sequenceid=53, filesize=47.0 K 2024-11-17T15:38:30,118 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/B/173682b39e1d4aa5b74b9bd7c9b83cc1 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/B/173682b39e1d4aa5b74b9bd7c9b83cc1 2024-11-17T15:38:30,121 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/B/173682b39e1d4aa5b74b9bd7c9b83cc1, entries=150, sequenceid=53, filesize=11.7 K 2024-11-17T15:38:30,122 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/C/9fe9b74d804c47d3b483e9fb7b172bf3 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/C/9fe9b74d804c47d3b483e9fb7b172bf3 2024-11-17T15:38:30,126 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/C/9fe9b74d804c47d3b483e9fb7b172bf3, entries=150, sequenceid=53, filesize=11.7 K 2024-11-17T15:38:30,126 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for fdca08c93967309fb9c5dcd33e10e7d5 in 1668ms, sequenceid=53, compaction requested=true 2024-11-17T15:38:30,127 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fdca08c93967309fb9c5dcd33e10e7d5: 2024-11-17T15:38:30,127 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fdca08c93967309fb9c5dcd33e10e7d5:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T15:38:30,127 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T15:38:30,127 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fdca08c93967309fb9c5dcd33e10e7d5:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T15:38:30,127 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T15:38:30,127 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T15:38:30,127 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fdca08c93967309fb9c5dcd33e10e7d5:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T15:38:30,127 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T15:38:30,127 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T15:38:30,128 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 127233 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T15:38:30,128 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T15:38:30,128 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1541): fdca08c93967309fb9c5dcd33e10e7d5/A is initiating minor compaction (all files) 2024-11-17T15:38:30,128 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1541): fdca08c93967309fb9c5dcd33e10e7d5/B is initiating minor compaction (all files) 2024-11-17T15:38:30,128 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fdca08c93967309fb9c5dcd33e10e7d5/B in TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. 2024-11-17T15:38:30,128 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fdca08c93967309fb9c5dcd33e10e7d5/A in TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. 2024-11-17T15:38:30,128 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/B/64ef4831062f49deb246b5f5662cdd79, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/B/2256f63d3a7b47dda6900cd2783f893a, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/B/173682b39e1d4aa5b74b9bd7c9b83cc1] into tmpdir=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp, totalSize=35.2 K 2024-11-17T15:38:30,128 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/A/a34332a6f86542279c4af1aa6e4e4dfe, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/A/1e8ab31ceecf4f959dfc79016730963e, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/A/c9029bd4d1a448febeb28f67eadc5494] into tmpdir=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp, totalSize=124.3 K 2024-11-17T15:38:30,128 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. 2024-11-17T15:38:30,128 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. files: [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/A/a34332a6f86542279c4af1aa6e4e4dfe, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/A/1e8ab31ceecf4f959dfc79016730963e, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/A/c9029bd4d1a448febeb28f67eadc5494] 2024-11-17T15:38:30,128 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting 64ef4831062f49deb246b5f5662cdd79, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731857905550 2024-11-17T15:38:30,128 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting a34332a6f86542279c4af1aa6e4e4dfe, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731857905550 2024-11-17T15:38:30,129 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting 2256f63d3a7b47dda6900cd2783f893a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731857905602 2024-11-17T15:38:30,129 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1e8ab31ceecf4f959dfc79016730963e, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731857905602 2024-11-17T15:38:30,129 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting 173682b39e1d4aa5b74b9bd7c9b83cc1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731857907807 2024-11-17T15:38:30,129 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting c9029bd4d1a448febeb28f67eadc5494, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731857907806 2024-11-17T15:38:30,136 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=fdca08c93967309fb9c5dcd33e10e7d5] 2024-11-17T15:38:30,137 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fdca08c93967309fb9c5dcd33e10e7d5#B#compaction#308 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T15:38:30,138 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/B/36262a5388854577b287964bcfb2c329 is 50, key is test_row_0/B:col10/1731857908458/Put/seqid=0 2024-11-17T15:38:30,141 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411170bfd9a202fab4dec8abbc47292d36b4a_fdca08c93967309fb9c5dcd33e10e7d5 store=[table=TestAcidGuarantees family=A region=fdca08c93967309fb9c5dcd33e10e7d5] 2024-11-17T15:38:30,142 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411170bfd9a202fab4dec8abbc47292d36b4a_fdca08c93967309fb9c5dcd33e10e7d5, store=[table=TestAcidGuarantees family=A region=fdca08c93967309fb9c5dcd33e10e7d5] 2024-11-17T15:38:30,142 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411170bfd9a202fab4dec8abbc47292d36b4a_fdca08c93967309fb9c5dcd33e10e7d5 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=fdca08c93967309fb9c5dcd33e10e7d5] 2024-11-17T15:38:30,143 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742189_1365 (size=12104) 2024-11-17T15:38:30,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742190_1366 (size=4469) 2024-11-17T15:38:30,153 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-17T15:38:30,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. 2024-11-17T15:38:30,154 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2902): Flushing fdca08c93967309fb9c5dcd33e10e7d5 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-17T15:38:30,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117ef2c1e03af9c49268d81aa9aecd35666_fdca08c93967309fb9c5dcd33e10e7d5 is 50, key is test_row_0/A:col10/1731857908516/Put/seqid=0 2024-11-17T15:38:30,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742191_1367 (size=12154) 2024-11-17T15:38:30,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-17T15:38:30,549 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/B/36262a5388854577b287964bcfb2c329 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/B/36262a5388854577b287964bcfb2c329 2024-11-17T15:38:30,553 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fdca08c93967309fb9c5dcd33e10e7d5/B of fdca08c93967309fb9c5dcd33e10e7d5 into 36262a5388854577b287964bcfb2c329(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T15:38:30,553 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fdca08c93967309fb9c5dcd33e10e7d5: 2024-11-17T15:38:30,554 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., storeName=fdca08c93967309fb9c5dcd33e10e7d5/B, priority=13, startTime=1731857910127; duration=0sec 2024-11-17T15:38:30,554 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T15:38:30,554 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fdca08c93967309fb9c5dcd33e10e7d5:B 2024-11-17T15:38:30,554 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fdca08c93967309fb9c5dcd33e10e7d5#A#compaction#309 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T15:38:30,554 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T15:38:30,554 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/A/070f58aa875e487d8b2605276feac97e is 175, key is test_row_0/A:col10/1731857908458/Put/seqid=0 2024-11-17T15:38:30,556 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T15:38:30,556 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1541): fdca08c93967309fb9c5dcd33e10e7d5/C is initiating minor compaction (all files) 2024-11-17T15:38:30,556 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fdca08c93967309fb9c5dcd33e10e7d5/C in TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. 2024-11-17T15:38:30,556 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/C/a261db2ea47f4841b60850d38b4687e9, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/C/46f8fdbd721040fc9fe4f70f89753664, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/C/9fe9b74d804c47d3b483e9fb7b172bf3] into tmpdir=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp, totalSize=35.2 K 2024-11-17T15:38:30,556 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting a261db2ea47f4841b60850d38b4687e9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731857905550 2024-11-17T15:38:30,557 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting 46f8fdbd721040fc9fe4f70f89753664, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731857905602 2024-11-17T15:38:30,557 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting 9fe9b74d804c47d3b483e9fb7b172bf3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731857907807 2024-11-17T15:38:30,564 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fdca08c93967309fb9c5dcd33e10e7d5#C#compaction#311 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T15:38:30,564 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/C/0e1a47e294724e26bc65376e765d0ecc is 50, key is test_row_0/C:col10/1731857908458/Put/seqid=0 2024-11-17T15:38:30,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742192_1368 (size=31058) 2024-11-17T15:38:30,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:30,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742193_1369 (size=12104) 2024-11-17T15:38:30,569 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117ef2c1e03af9c49268d81aa9aecd35666_fdca08c93967309fb9c5dcd33e10e7d5 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117ef2c1e03af9c49268d81aa9aecd35666_fdca08c93967309fb9c5dcd33e10e7d5 2024-11-17T15:38:30,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/A/81c4b313a35c4ec392a6678b7f6e049b, store: [table=TestAcidGuarantees family=A region=fdca08c93967309fb9c5dcd33e10e7d5] 2024-11-17T15:38:30,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/A/81c4b313a35c4ec392a6678b7f6e049b is 175, key is test_row_0/A:col10/1731857908516/Put/seqid=0 2024-11-17T15:38:30,572 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/A/070f58aa875e487d8b2605276feac97e as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/A/070f58aa875e487d8b2605276feac97e 2024-11-17T15:38:30,578 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fdca08c93967309fb9c5dcd33e10e7d5/A of fdca08c93967309fb9c5dcd33e10e7d5 into 070f58aa875e487d8b2605276feac97e(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T15:38:30,578 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fdca08c93967309fb9c5dcd33e10e7d5: 2024-11-17T15:38:30,578 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., storeName=fdca08c93967309fb9c5dcd33e10e7d5/A, priority=13, startTime=1731857910127; duration=0sec 2024-11-17T15:38:30,578 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T15:38:30,578 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fdca08c93967309fb9c5dcd33e10e7d5:A 2024-11-17T15:38:30,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742194_1370 (size=30955) 2024-11-17T15:38:30,587 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=76, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/A/81c4b313a35c4ec392a6678b7f6e049b 2024-11-17T15:38:30,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/B/ab3d23baf6884f65956c9dddcd3ecff7 is 50, key is test_row_0/B:col10/1731857908516/Put/seqid=0 2024-11-17T15:38:30,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742195_1371 (size=12001) 2024-11-17T15:38:30,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(8855): Flush requested on fdca08c93967309fb9c5dcd33e10e7d5 2024-11-17T15:38:30,708 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. as already flushing 2024-11-17T15:38:30,734 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:30,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49652 deadline: 1731857970722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:30,734 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:30,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49654 deadline: 1731857970723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:30,735 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:30,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49700 deadline: 1731857970729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:30,735 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:30,735 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:30,735 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:30,735 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 7a780d55532c,32955,1731857800648 for 'test_row_1' in fdca08c93967309fb9c5dcd33e10e7d5 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2216 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T15:38:30,735 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:30,735 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:30,735 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:30,735 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:30,735 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:30,735 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:30,740 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:30,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49680 deadline: 1731857970730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:30,740 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:30,740 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:30,740 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:30,741 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:30,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49690 deadline: 1731857970734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:30,741 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:30,741 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:30,741 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:30,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-17T15:38:30,848 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:30,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49654 deadline: 1731857970843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:30,848 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:30,848 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:30,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49700 deadline: 1731857970843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:30,848 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:30,848 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:30,849 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:30,849 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:30,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49680 deadline: 1731857970843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:30,849 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:30,849 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:30,849 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:30,849 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:30,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49690 deadline: 1731857970843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:30,849 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:30,849 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:30,849 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:30,849 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:30,849 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:30,974 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/C/0e1a47e294724e26bc65376e765d0ecc as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/C/0e1a47e294724e26bc65376e765d0ecc 2024-11-17T15:38:30,978 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fdca08c93967309fb9c5dcd33e10e7d5/C of fdca08c93967309fb9c5dcd33e10e7d5 into 0e1a47e294724e26bc65376e765d0ecc(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T15:38:30,978 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fdca08c93967309fb9c5dcd33e10e7d5: 2024-11-17T15:38:30,978 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., storeName=fdca08c93967309fb9c5dcd33e10e7d5/C, priority=13, startTime=1731857910127; duration=0sec 2024-11-17T15:38:30,978 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T15:38:30,978 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fdca08c93967309fb9c5dcd33e10e7d5:C 2024-11-17T15:38:30,998 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/B/ab3d23baf6884f65956c9dddcd3ecff7 2024-11-17T15:38:31,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/C/a30acc0c07b24206a00d4f2f0ceef2f9 is 50, key is test_row_0/C:col10/1731857908516/Put/seqid=0 2024-11-17T15:38:31,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742196_1372 (size=12001) 2024-11-17T15:38:31,060 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:31,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49654 deadline: 1731857971053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:31,061 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:31,061 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:31,061 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:31,061 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:31,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49700 deadline: 1731857971054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:31,061 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:31,061 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:31,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49680 deadline: 1731857971054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:31,061 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:31,061 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:31,062 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:31,062 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:31,062 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:31,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49690 deadline: 1731857971054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:31,062 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:31,062 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:31,062 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:31,062 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:31,368 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:31,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49654 deadline: 1731857971363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:31,369 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:31,369 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:31,369 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:31,380 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:31,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49700 deadline: 1731857971373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:31,381 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:31,381 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:31,381 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:31,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49680 deadline: 1731857971373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:31,381 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:31,381 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:31,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49690 deadline: 1731857971373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:31,381 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:31,381 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:31,381 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:31,381 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:31,381 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:31,381 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:31,409 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/C/a30acc0c07b24206a00d4f2f0ceef2f9 2024-11-17T15:38:31,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/A/81c4b313a35c4ec392a6678b7f6e049b as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/A/81c4b313a35c4ec392a6678b7f6e049b 2024-11-17T15:38:31,416 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/A/81c4b313a35c4ec392a6678b7f6e049b, entries=150, sequenceid=76, filesize=30.2 K 2024-11-17T15:38:31,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/B/ab3d23baf6884f65956c9dddcd3ecff7 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/B/ab3d23baf6884f65956c9dddcd3ecff7 2024-11-17T15:38:31,420 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/B/ab3d23baf6884f65956c9dddcd3ecff7, entries=150, sequenceid=76, filesize=11.7 K 2024-11-17T15:38:31,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/C/a30acc0c07b24206a00d4f2f0ceef2f9 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/C/a30acc0c07b24206a00d4f2f0ceef2f9 2024-11-17T15:38:31,424 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/C/a30acc0c07b24206a00d4f2f0ceef2f9, entries=150, sequenceid=76, filesize=11.7 K 2024-11-17T15:38:31,425 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for fdca08c93967309fb9c5dcd33e10e7d5 in 1271ms, sequenceid=76, compaction requested=false 2024-11-17T15:38:31,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2603): Flush status journal for fdca08c93967309fb9c5dcd33e10e7d5: 2024-11-17T15:38:31,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. 2024-11-17T15:38:31,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=106 2024-11-17T15:38:31,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.HMaster(4169): Remote procedure done, pid=106 2024-11-17T15:38:31,427 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-11-17T15:38:31,428 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7310 sec 2024-11-17T15:38:31,429 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees in 1.7350 sec 2024-11-17T15:38:31,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-17T15:38:31,843 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T15:38:31,843 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T15:38:31,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees 2024-11-17T15:38:31,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-17T15:38:31,845 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T15:38:31,846 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T15:38:31,846 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T15:38:31,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(8855): Flush requested on fdca08c93967309fb9c5dcd33e10e7d5 2024-11-17T15:38:31,879 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T15:38:31,879 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fdca08c93967309fb9c5dcd33e10e7d5 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-17T15:38:31,889 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117f39cc1839799464b8841930d066c83a8_fdca08c93967309fb9c5dcd33e10e7d5 is 50, key is test_row_0/A:col10/1731857910733/Put/seqid=0 2024-11-17T15:38:31,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742197_1373 (size=14594) 2024-11-17T15:38:31,944 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:31,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49680 deadline: 1731857971933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:31,945 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:31,945 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:31,945 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:31,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49690 deadline: 1731857971934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:31,945 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:31,945 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:31,945 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:31,945 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:31,947 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:31,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49700 deadline: 1731857971935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:31,947 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:31,947 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:31,947 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:31,948 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:31,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49654 deadline: 1731857971945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:31,949 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:31,949 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:31,949 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:31,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-17T15:38:31,998 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-17T15:38:31,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. 2024-11-17T15:38:31,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. as already flushing 2024-11-17T15:38:31,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. 2024-11-17T15:38:31,999 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => fdca08c93967309fb9c5dcd33e10e7d5, NAME => 'TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:31,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => fdca08c93967309fb9c5dcd33e10e7d5, NAME => 'TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:31,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fdca08c93967309fb9c5dcd33e10e7d5, NAME => 'TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fdca08c93967309fb9c5dcd33e10e7d5, NAME => 'TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:32,057 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:32,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49680 deadline: 1731857972053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:32,057 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:32,057 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:32,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49690 deadline: 1731857972053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:32,057 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:32,057 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:32,057 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:32,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49654 deadline: 1731857972053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:32,058 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:32,058 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:32,058 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:32,058 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:32,058 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:32,058 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:32,058 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:32,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49700 deadline: 1731857972053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:32,058 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:32,058 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:32,058 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:32,151 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-17T15:38:32,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. 2024-11-17T15:38:32,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. as already flushing 2024-11-17T15:38:32,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. 2024-11-17T15:38:32,152 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => fdca08c93967309fb9c5dcd33e10e7d5, NAME => 'TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:32,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => fdca08c93967309fb9c5dcd33e10e7d5, NAME => 'TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:32,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fdca08c93967309fb9c5dcd33e10e7d5, NAME => 'TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fdca08c93967309fb9c5dcd33e10e7d5, NAME => 'TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:32,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-17T15:38:32,272 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:32,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49680 deadline: 1731857972263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:32,272 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:32,272 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:32,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49690 deadline: 1731857972263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:32,272 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:32,272 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:32,273 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:32,273 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:32,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49700 deadline: 1731857972263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:32,273 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:32,273 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:32,273 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:32,273 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:32,273 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:32,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49654 deadline: 1731857972263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:32,273 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:32,273 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:32,273 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:32,273 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:32,293 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:32,297 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117f39cc1839799464b8841930d066c83a8_fdca08c93967309fb9c5dcd33e10e7d5 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117f39cc1839799464b8841930d066c83a8_fdca08c93967309fb9c5dcd33e10e7d5 2024-11-17T15:38:32,298 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/A/ea51521fd44745ed8387b009a19f365a, store: [table=TestAcidGuarantees family=A region=fdca08c93967309fb9c5dcd33e10e7d5] 2024-11-17T15:38:32,298 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/A/ea51521fd44745ed8387b009a19f365a is 175, key is test_row_0/A:col10/1731857910733/Put/seqid=0 2024-11-17T15:38:32,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742198_1374 (size=39549) 2024-11-17T15:38:32,304 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-17T15:38:32,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. 2024-11-17T15:38:32,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. as already flushing 2024-11-17T15:38:32,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. 2024-11-17T15:38:32,304 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => fdca08c93967309fb9c5dcd33e10e7d5, NAME => 'TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:32,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => fdca08c93967309fb9c5dcd33e10e7d5, NAME => 'TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:32,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fdca08c93967309fb9c5dcd33e10e7d5, NAME => 'TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fdca08c93967309fb9c5dcd33e10e7d5, NAME => 'TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:32,457 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-17T15:38:32,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. 2024-11-17T15:38:32,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. as already flushing 2024-11-17T15:38:32,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. 2024-11-17T15:38:32,457 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => fdca08c93967309fb9c5dcd33e10e7d5, NAME => 'TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:32,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => fdca08c93967309fb9c5dcd33e10e7d5, NAME => 'TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:32,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fdca08c93967309fb9c5dcd33e10e7d5, NAME => 'TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fdca08c93967309fb9c5dcd33e10e7d5, NAME => 'TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:32,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-17T15:38:32,587 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:32,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49680 deadline: 1731857972583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:32,587 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:32,587 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:32,587 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:32,593 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:32,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49690 deadline: 1731857972583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:32,593 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:32,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49700 deadline: 1731857972583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:32,593 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:32,593 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:32,593 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:32,593 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:32,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49654 deadline: 1731857972583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:32,593 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:32,593 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:32,593 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:32,593 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:32,593 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:32,593 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:32,610 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-17T15:38:32,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. 2024-11-17T15:38:32,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. as already flushing 2024-11-17T15:38:32,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. 2024-11-17T15:38:32,610 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => fdca08c93967309fb9c5dcd33e10e7d5, NAME => 'TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:32,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => fdca08c93967309fb9c5dcd33e10e7d5, NAME => 'TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:32,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fdca08c93967309fb9c5dcd33e10e7d5, NAME => 'TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fdca08c93967309fb9c5dcd33e10e7d5, NAME => 'TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:32,702 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=93, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/A/ea51521fd44745ed8387b009a19f365a 2024-11-17T15:38:32,709 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/B/ed776f37a2e24f4fa54460acfcb1ec72 is 50, key is test_row_0/B:col10/1731857910733/Put/seqid=0 2024-11-17T15:38:32,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742199_1375 (size=12001) 2024-11-17T15:38:32,758 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:32,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49652 deadline: 1731857972753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:32,758 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 7a780d55532c,32955,1731857800648 for 'test_row_1' in fdca08c93967309fb9c5dcd33e10e7d5 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4240 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T15:38:32,759 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:32,759 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:32,759 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:32,762 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-17T15:38:32,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. 2024-11-17T15:38:32,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. as already flushing 2024-11-17T15:38:32,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. 2024-11-17T15:38:32,763 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => fdca08c93967309fb9c5dcd33e10e7d5, NAME => 'TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:32,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => fdca08c93967309fb9c5dcd33e10e7d5, NAME => 'TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:32,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fdca08c93967309fb9c5dcd33e10e7d5, NAME => 'TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fdca08c93967309fb9c5dcd33e10e7d5, NAME => 'TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:32,915 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-17T15:38:32,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. 2024-11-17T15:38:32,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. as already flushing 2024-11-17T15:38:32,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. 2024-11-17T15:38:32,916 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => fdca08c93967309fb9c5dcd33e10e7d5, NAME => 'TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:32,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => fdca08c93967309fb9c5dcd33e10e7d5, NAME => 'TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:32,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fdca08c93967309fb9c5dcd33e10e7d5, NAME => 'TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fdca08c93967309fb9c5dcd33e10e7d5, NAME => 'TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:32,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-17T15:38:33,068 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-17T15:38:33,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. 2024-11-17T15:38:33,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. as already flushing 2024-11-17T15:38:33,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. 2024-11-17T15:38:33,069 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => fdca08c93967309fb9c5dcd33e10e7d5, NAME => 'TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:33,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => fdca08c93967309fb9c5dcd33e10e7d5, NAME => 'TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:33,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fdca08c93967309fb9c5dcd33e10e7d5, NAME => 'TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fdca08c93967309fb9c5dcd33e10e7d5, NAME => 'TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:33,097 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:33,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49680 deadline: 1731857973093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:33,097 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:33,098 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:33,098 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:33,109 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:33,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49690 deadline: 1731857973103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:33,109 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:33,109 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:33,110 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:33,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49700 deadline: 1731857973103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:33,110 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:33,110 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:33,110 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:33,110 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:33,110 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:33,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49654 deadline: 1731857973103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:33,110 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:33,110 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:33,110 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:33,115 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/B/ed776f37a2e24f4fa54460acfcb1ec72 2024-11-17T15:38:33,122 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/C/6a9cf3c9ed5d4b1abb0437cf3a05aac9 is 50, key is test_row_0/C:col10/1731857910733/Put/seqid=0 2024-11-17T15:38:33,127 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742200_1376 (size=12001) 2024-11-17T15:38:33,221 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-17T15:38:33,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. 2024-11-17T15:38:33,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. as already flushing 2024-11-17T15:38:33,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. 2024-11-17T15:38:33,222 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => fdca08c93967309fb9c5dcd33e10e7d5, NAME => 'TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:33,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => fdca08c93967309fb9c5dcd33e10e7d5, NAME => 'TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:33,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fdca08c93967309fb9c5dcd33e10e7d5, NAME => 'TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fdca08c93967309fb9c5dcd33e10e7d5, NAME => 'TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:33,374 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-17T15:38:33,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. 2024-11-17T15:38:33,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. as already flushing 2024-11-17T15:38:33,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. 2024-11-17T15:38:33,375 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => fdca08c93967309fb9c5dcd33e10e7d5, NAME => 'TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:33,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => fdca08c93967309fb9c5dcd33e10e7d5, NAME => 'TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:33,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fdca08c93967309fb9c5dcd33e10e7d5, NAME => 'TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fdca08c93967309fb9c5dcd33e10e7d5, NAME => 'TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:33,526 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-17T15:38:33,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. 2024-11-17T15:38:33,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. as already flushing 2024-11-17T15:38:33,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. 2024-11-17T15:38:33,527 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => fdca08c93967309fb9c5dcd33e10e7d5, NAME => 'TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:33,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => fdca08c93967309fb9c5dcd33e10e7d5, NAME => 'TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:33,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fdca08c93967309fb9c5dcd33e10e7d5, NAME => 'TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fdca08c93967309fb9c5dcd33e10e7d5, NAME => 'TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:33,528 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/C/6a9cf3c9ed5d4b1abb0437cf3a05aac9 2024-11-17T15:38:33,535 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/A/ea51521fd44745ed8387b009a19f365a as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/A/ea51521fd44745ed8387b009a19f365a 2024-11-17T15:38:33,538 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/A/ea51521fd44745ed8387b009a19f365a, entries=200, sequenceid=93, filesize=38.6 K 2024-11-17T15:38:33,539 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/B/ed776f37a2e24f4fa54460acfcb1ec72 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/B/ed776f37a2e24f4fa54460acfcb1ec72 2024-11-17T15:38:33,542 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/B/ed776f37a2e24f4fa54460acfcb1ec72, entries=150, sequenceid=93, filesize=11.7 K 2024-11-17T15:38:33,543 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/C/6a9cf3c9ed5d4b1abb0437cf3a05aac9 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/C/6a9cf3c9ed5d4b1abb0437cf3a05aac9 2024-11-17T15:38:33,547 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/C/6a9cf3c9ed5d4b1abb0437cf3a05aac9, entries=150, sequenceid=93, filesize=11.7 K 2024-11-17T15:38:33,548 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for fdca08c93967309fb9c5dcd33e10e7d5 in 1669ms, sequenceid=93, compaction requested=true 2024-11-17T15:38:33,548 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fdca08c93967309fb9c5dcd33e10e7d5: 2024-11-17T15:38:33,548 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fdca08c93967309fb9c5dcd33e10e7d5:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T15:38:33,548 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T15:38:33,548 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T15:38:33,548 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T15:38:33,549 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fdca08c93967309fb9c5dcd33e10e7d5:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T15:38:33,549 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T15:38:33,549 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fdca08c93967309fb9c5dcd33e10e7d5:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T15:38:33,549 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T15:38:33,549 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T15:38:33,549 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101562 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T15:38:33,549 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1541): fdca08c93967309fb9c5dcd33e10e7d5/B is initiating minor compaction (all files) 2024-11-17T15:38:33,549 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1541): fdca08c93967309fb9c5dcd33e10e7d5/A is initiating minor compaction (all files) 2024-11-17T15:38:33,549 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fdca08c93967309fb9c5dcd33e10e7d5/A in TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. 2024-11-17T15:38:33,549 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fdca08c93967309fb9c5dcd33e10e7d5/B in TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. 2024-11-17T15:38:33,550 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/B/36262a5388854577b287964bcfb2c329, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/B/ab3d23baf6884f65956c9dddcd3ecff7, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/B/ed776f37a2e24f4fa54460acfcb1ec72] into tmpdir=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp, totalSize=35.3 K 2024-11-17T15:38:33,550 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/A/070f58aa875e487d8b2605276feac97e, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/A/81c4b313a35c4ec392a6678b7f6e049b, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/A/ea51521fd44745ed8387b009a19f365a] into tmpdir=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp, totalSize=99.2 K 2024-11-17T15:38:33,550 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. 2024-11-17T15:38:33,550 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. files: [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/A/070f58aa875e487d8b2605276feac97e, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/A/81c4b313a35c4ec392a6678b7f6e049b, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/A/ea51521fd44745ed8387b009a19f365a] 2024-11-17T15:38:33,550 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting 36262a5388854577b287964bcfb2c329, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731857907807 2024-11-17T15:38:33,550 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 070f58aa875e487d8b2605276feac97e, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731857907807 2024-11-17T15:38:33,550 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting ab3d23baf6884f65956c9dddcd3ecff7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731857908510 2024-11-17T15:38:33,550 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 81c4b313a35c4ec392a6678b7f6e049b, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731857908510 2024-11-17T15:38:33,550 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting ed776f37a2e24f4fa54460acfcb1ec72, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731857910728 2024-11-17T15:38:33,551 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting ea51521fd44745ed8387b009a19f365a, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731857910728 2024-11-17T15:38:33,558 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=fdca08c93967309fb9c5dcd33e10e7d5] 2024-11-17T15:38:33,559 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fdca08c93967309fb9c5dcd33e10e7d5#B#compaction#317 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T15:38:33,560 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/B/14c24457fdf248e1abefb22cc1722838 is 50, key is test_row_0/B:col10/1731857910733/Put/seqid=0 2024-11-17T15:38:33,560 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111779eccb1085684cb38d478388de02de2f_fdca08c93967309fb9c5dcd33e10e7d5 store=[table=TestAcidGuarantees family=A region=fdca08c93967309fb9c5dcd33e10e7d5] 2024-11-17T15:38:33,562 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111779eccb1085684cb38d478388de02de2f_fdca08c93967309fb9c5dcd33e10e7d5, store=[table=TestAcidGuarantees family=A region=fdca08c93967309fb9c5dcd33e10e7d5] 2024-11-17T15:38:33,562 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111779eccb1085684cb38d478388de02de2f_fdca08c93967309fb9c5dcd33e10e7d5 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=fdca08c93967309fb9c5dcd33e10e7d5] 2024-11-17T15:38:33,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742201_1377 (size=12207) 2024-11-17T15:38:33,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742202_1378 (size=4469) 2024-11-17T15:38:33,680 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-17T15:38:33,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. 2024-11-17T15:38:33,680 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2902): Flushing fdca08c93967309fb9c5dcd33e10e7d5 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-17T15:38:33,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117968eec3471234322a3f5a35dc4399c59_fdca08c93967309fb9c5dcd33e10e7d5 is 50, key is test_row_0/A:col10/1731857911930/Put/seqid=0 2024-11-17T15:38:33,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742203_1379 (size=12154) 2024-11-17T15:38:33,972 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fdca08c93967309fb9c5dcd33e10e7d5#A#compaction#318 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T15:38:33,972 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/A/06f5c7669a334db7b2425a36a5e0b8d2 is 175, key is test_row_0/A:col10/1731857910733/Put/seqid=0 2024-11-17T15:38:33,973 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/B/14c24457fdf248e1abefb22cc1722838 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/B/14c24457fdf248e1abefb22cc1722838 2024-11-17T15:38:33,976 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742204_1380 (size=31161) 2024-11-17T15:38:33,979 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fdca08c93967309fb9c5dcd33e10e7d5/B of fdca08c93967309fb9c5dcd33e10e7d5 into 14c24457fdf248e1abefb22cc1722838(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T15:38:33,979 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fdca08c93967309fb9c5dcd33e10e7d5: 2024-11-17T15:38:33,979 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., storeName=fdca08c93967309fb9c5dcd33e10e7d5/B, priority=13, startTime=1731857913548; duration=0sec 2024-11-17T15:38:33,979 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T15:38:33,979 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fdca08c93967309fb9c5dcd33e10e7d5:B 2024-11-17T15:38:33,979 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T15:38:33,981 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T15:38:33,981 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1541): fdca08c93967309fb9c5dcd33e10e7d5/C is initiating minor compaction (all files) 2024-11-17T15:38:33,981 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fdca08c93967309fb9c5dcd33e10e7d5/C in TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. 2024-11-17T15:38:33,981 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/C/0e1a47e294724e26bc65376e765d0ecc, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/C/a30acc0c07b24206a00d4f2f0ceef2f9, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/C/6a9cf3c9ed5d4b1abb0437cf3a05aac9] into tmpdir=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp, totalSize=35.3 K 2024-11-17T15:38:33,981 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting 0e1a47e294724e26bc65376e765d0ecc, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731857907807 2024-11-17T15:38:33,981 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting a30acc0c07b24206a00d4f2f0ceef2f9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731857908510 2024-11-17T15:38:33,982 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting 6a9cf3c9ed5d4b1abb0437cf3a05aac9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731857910728 2024-11-17T15:38:33,988 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fdca08c93967309fb9c5dcd33e10e7d5#C#compaction#320 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T15:38:33,989 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/C/ffd5f307d98949068959f8e860c04ef2 is 50, key is test_row_0/C:col10/1731857910733/Put/seqid=0 2024-11-17T15:38:33,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742205_1381 (size=12207) 2024-11-17T15:38:33,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-17T15:38:34,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:34,095 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117968eec3471234322a3f5a35dc4399c59_fdca08c93967309fb9c5dcd33e10e7d5 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117968eec3471234322a3f5a35dc4399c59_fdca08c93967309fb9c5dcd33e10e7d5 2024-11-17T15:38:34,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/A/2fae58e5cf364c1aab9154350291067f, store: [table=TestAcidGuarantees family=A region=fdca08c93967309fb9c5dcd33e10e7d5] 2024-11-17T15:38:34,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/A/2fae58e5cf364c1aab9154350291067f is 175, key is test_row_0/A:col10/1731857911930/Put/seqid=0 2024-11-17T15:38:34,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(8855): Flush requested on fdca08c93967309fb9c5dcd33e10e7d5 2024-11-17T15:38:34,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742206_1382 (size=30955) 2024-11-17T15:38:34,114 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. as already flushing 2024-11-17T15:38:34,129 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=115, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/A/2fae58e5cf364c1aab9154350291067f 2024-11-17T15:38:34,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/B/858a9b0419cc444a9f6f9fb54aaf1938 is 50, key is test_row_0/B:col10/1731857911930/Put/seqid=0 2024-11-17T15:38:34,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742207_1383 (size=12001) 2024-11-17T15:38:34,166 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:34,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49690 deadline: 1731857974153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:34,167 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:34,167 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:34,167 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:34,167 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:34,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49700 deadline: 1731857974155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:34,167 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:34,167 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:34,167 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:34,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49654 deadline: 1731857974156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:34,167 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:34,168 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:34,168 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:34,168 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:34,172 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:34,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49680 deadline: 1731857974158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:34,173 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:34,173 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:34,173 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:34,278 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:34,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49690 deadline: 1731857974273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:34,278 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:34,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49700 deadline: 1731857974273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:34,278 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:34,278 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:34,278 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:34,278 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:34,278 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:34,278 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:34,278 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:34,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49654 deadline: 1731857974273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:34,279 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:34,279 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:34,279 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:34,291 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:34,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49680 deadline: 1731857974283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:34,292 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:34,292 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:34,292 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:34,382 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/A/06f5c7669a334db7b2425a36a5e0b8d2 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/A/06f5c7669a334db7b2425a36a5e0b8d2 2024-11-17T15:38:34,387 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fdca08c93967309fb9c5dcd33e10e7d5/A of fdca08c93967309fb9c5dcd33e10e7d5 into 06f5c7669a334db7b2425a36a5e0b8d2(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T15:38:34,387 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fdca08c93967309fb9c5dcd33e10e7d5: 2024-11-17T15:38:34,387 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., storeName=fdca08c93967309fb9c5dcd33e10e7d5/A, priority=13, startTime=1731857913548; duration=0sec 2024-11-17T15:38:34,387 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T15:38:34,387 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fdca08c93967309fb9c5dcd33e10e7d5:A 2024-11-17T15:38:34,403 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/C/ffd5f307d98949068959f8e860c04ef2 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/C/ffd5f307d98949068959f8e860c04ef2 2024-11-17T15:38:34,408 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fdca08c93967309fb9c5dcd33e10e7d5/C of fdca08c93967309fb9c5dcd33e10e7d5 into ffd5f307d98949068959f8e860c04ef2(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T15:38:34,408 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fdca08c93967309fb9c5dcd33e10e7d5: 2024-11-17T15:38:34,408 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., storeName=fdca08c93967309fb9c5dcd33e10e7d5/C, priority=13, startTime=1731857913549; duration=0sec 2024-11-17T15:38:34,408 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T15:38:34,408 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fdca08c93967309fb9c5dcd33e10e7d5:C 2024-11-17T15:38:34,494 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:34,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49690 deadline: 1731857974483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:34,495 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:34,495 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:34,495 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:34,495 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:34,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49700 deadline: 1731857974483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:34,495 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:34,495 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:34,495 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:34,495 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:34,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49654 deadline: 1731857974483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:34,496 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:34,496 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:34,496 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:34,511 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:34,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49680 deadline: 1731857974503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:34,511 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:34,511 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:34,511 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:34,550 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/B/858a9b0419cc444a9f6f9fb54aaf1938 2024-11-17T15:38:34,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/C/b6bbd5aab24c49e9a68e82b9991dc29c is 50, key is test_row_0/C:col10/1731857911930/Put/seqid=0 2024-11-17T15:38:34,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742208_1384 (size=12001) 2024-11-17T15:38:34,811 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:34,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49690 deadline: 1731857974803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:34,811 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:34,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49700 deadline: 1731857974803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:34,812 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:34,812 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:34,812 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:34,812 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:34,812 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:34,812 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:34,812 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:34,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49654 deadline: 1731857974803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:34,812 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:34,812 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:34,812 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:34,830 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:34,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49680 deadline: 1731857974823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:34,831 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:34,831 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:34,831 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:34,962 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/C/b6bbd5aab24c49e9a68e82b9991dc29c 2024-11-17T15:38:34,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/A/2fae58e5cf364c1aab9154350291067f as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/A/2fae58e5cf364c1aab9154350291067f 2024-11-17T15:38:34,971 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/A/2fae58e5cf364c1aab9154350291067f, entries=150, sequenceid=115, filesize=30.2 K 2024-11-17T15:38:34,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/B/858a9b0419cc444a9f6f9fb54aaf1938 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/B/858a9b0419cc444a9f6f9fb54aaf1938 2024-11-17T15:38:34,976 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/B/858a9b0419cc444a9f6f9fb54aaf1938, entries=150, sequenceid=115, filesize=11.7 K 2024-11-17T15:38:34,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/C/b6bbd5aab24c49e9a68e82b9991dc29c as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/C/b6bbd5aab24c49e9a68e82b9991dc29c 2024-11-17T15:38:34,981 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/C/b6bbd5aab24c49e9a68e82b9991dc29c, entries=150, sequenceid=115, filesize=11.7 K 2024-11-17T15:38:34,982 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for fdca08c93967309fb9c5dcd33e10e7d5 in 1302ms, sequenceid=115, compaction requested=false 2024-11-17T15:38:34,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2603): Flush status journal for fdca08c93967309fb9c5dcd33e10e7d5: 2024-11-17T15:38:34,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. 2024-11-17T15:38:34,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=108 2024-11-17T15:38:34,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.HMaster(4169): Remote procedure done, pid=108 2024-11-17T15:38:34,985 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-11-17T15:38:34,985 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.1380 sec 2024-11-17T15:38:34,988 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees in 3.1430 sec 2024-11-17T15:38:35,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(8855): Flush requested on fdca08c93967309fb9c5dcd33e10e7d5 2024-11-17T15:38:35,325 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T15:38:35,325 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fdca08c93967309fb9c5dcd33e10e7d5 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-17T15:38:35,341 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117261771ac4b7040978e756f0bc2b3f4ac_fdca08c93967309fb9c5dcd33e10e7d5 is 50, key is test_row_0/A:col10/1731857915324/Put/seqid=0 2024-11-17T15:38:35,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742209_1385 (size=14694) 2024-11-17T15:38:35,380 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:35,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49654 deadline: 1731857975368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:35,381 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:35,381 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:35,381 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:35,391 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:35,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49690 deadline: 1731857975378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:35,391 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:35,391 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:35,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49700 deadline: 1731857975379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:35,391 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:35,391 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:35,391 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:35,391 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:35,392 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:35,394 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:35,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49680 deadline: 1731857975381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:35,394 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:35,394 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:35,394 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:35,489 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:35,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49654 deadline: 1731857975485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:35,489 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:35,489 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:35,489 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:35,502 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:35,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49690 deadline: 1731857975493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:35,503 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:35,503 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:35,503 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:35,513 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:35,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49700 deadline: 1731857975504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:35,513 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:35,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49680 deadline: 1731857975504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:35,515 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:35,515 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:35,515 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:35,515 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:35,515 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:35,515 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:35,704 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:35,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49654 deadline: 1731857975693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:35,705 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:35,705 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:35,705 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:35,719 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:35,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49690 deadline: 1731857975712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:35,720 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:35,720 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:35,720 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:35,734 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:35,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49700 deadline: 1731857975723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:35,734 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:35,734 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:35,734 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:35,734 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:35,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49680 deadline: 1731857975723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:35,734 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:35,735 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:35,735 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:35,751 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:35,755 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117261771ac4b7040978e756f0bc2b3f4ac_fdca08c93967309fb9c5dcd33e10e7d5 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117261771ac4b7040978e756f0bc2b3f4ac_fdca08c93967309fb9c5dcd33e10e7d5 2024-11-17T15:38:35,756 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/A/ad6382eeb92b43d282cea63b5c453a4d, store: [table=TestAcidGuarantees family=A region=fdca08c93967309fb9c5dcd33e10e7d5] 2024-11-17T15:38:35,756 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/A/ad6382eeb92b43d282cea63b5c453a4d is 175, key is test_row_0/A:col10/1731857915324/Put/seqid=0 2024-11-17T15:38:35,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742210_1386 (size=39649) 2024-11-17T15:38:36,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-17T15:38:36,014 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T15:38:36,014 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T15:38:36,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees 2024-11-17T15:38:36,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-17T15:38:36,016 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T15:38:36,017 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T15:38:36,017 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T15:38:36,022 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:36,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49654 deadline: 1731857976013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:36,023 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:36,023 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:36,023 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:36,029 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:36,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49690 deadline: 1731857976022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:36,029 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:36,029 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:36,029 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:36,050 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:36,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49700 deadline: 1731857976043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:36,050 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:36,050 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:36,050 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:36,050 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:36,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49680 deadline: 1731857976043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:36,051 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:36,051 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:36,051 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:36,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-17T15:38:36,166 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=133, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/A/ad6382eeb92b43d282cea63b5c453a4d 2024-11-17T15:38:36,169 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-17T15:38:36,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. 2024-11-17T15:38:36,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. as already flushing 2024-11-17T15:38:36,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. 2024-11-17T15:38:36,170 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => fdca08c93967309fb9c5dcd33e10e7d5, NAME => 'TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:36,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => fdca08c93967309fb9c5dcd33e10e7d5, NAME => 'TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:36,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fdca08c93967309fb9c5dcd33e10e7d5, NAME => 'TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fdca08c93967309fb9c5dcd33e10e7d5, NAME => 'TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:36,178 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/B/ced4749f804a4d54b349240bd2592b4c is 50, key is test_row_0/B:col10/1731857915324/Put/seqid=0 2024-11-17T15:38:36,189 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742211_1387 (size=12101) 2024-11-17T15:38:36,190 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/B/ced4749f804a4d54b349240bd2592b4c 2024-11-17T15:38:36,200 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/C/f2ab4b89981a44aaa1ad9c44815c0289 is 50, key is test_row_0/C:col10/1731857915324/Put/seqid=0 2024-11-17T15:38:36,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742212_1388 (size=12101) 2024-11-17T15:38:36,322 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-17T15:38:36,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. 2024-11-17T15:38:36,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. as already flushing 2024-11-17T15:38:36,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. 2024-11-17T15:38:36,323 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => fdca08c93967309fb9c5dcd33e10e7d5, NAME => 'TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:36,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => fdca08c93967309fb9c5dcd33e10e7d5, NAME => 'TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:36,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fdca08c93967309fb9c5dcd33e10e7d5, NAME => 'TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fdca08c93967309fb9c5dcd33e10e7d5, NAME => 'TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:36,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-17T15:38:36,476 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-17T15:38:36,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. 2024-11-17T15:38:36,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. as already flushing 2024-11-17T15:38:36,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. 2024-11-17T15:38:36,476 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => fdca08c93967309fb9c5dcd33e10e7d5, NAME => 'TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:36,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => fdca08c93967309fb9c5dcd33e10e7d5, NAME => 'TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:36,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fdca08c93967309fb9c5dcd33e10e7d5, NAME => 'TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fdca08c93967309fb9c5dcd33e10e7d5, NAME => 'TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:36,539 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:36,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49654 deadline: 1731857976533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:36,540 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:36,540 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:36,540 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:36,554 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:36,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49690 deadline: 1731857976543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:36,554 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:36,554 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:36,554 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:36,563 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:36,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49700 deadline: 1731857976553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:36,564 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:36,564 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:36,564 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:36,573 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:36,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49680 deadline: 1731857976563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:36,573 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:36,573 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:36,573 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:36,610 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/C/f2ab4b89981a44aaa1ad9c44815c0289 2024-11-17T15:38:36,615 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/A/ad6382eeb92b43d282cea63b5c453a4d as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/A/ad6382eeb92b43d282cea63b5c453a4d 2024-11-17T15:38:36,619 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/A/ad6382eeb92b43d282cea63b5c453a4d, entries=200, sequenceid=133, filesize=38.7 K 2024-11-17T15:38:36,619 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/B/ced4749f804a4d54b349240bd2592b4c as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/B/ced4749f804a4d54b349240bd2592b4c 2024-11-17T15:38:36,625 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/B/ced4749f804a4d54b349240bd2592b4c, entries=150, sequenceid=133, filesize=11.8 K 2024-11-17T15:38:36,626 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/C/f2ab4b89981a44aaa1ad9c44815c0289 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/C/f2ab4b89981a44aaa1ad9c44815c0289 2024-11-17T15:38:36,629 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-17T15:38:36,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. 2024-11-17T15:38:36,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. as already flushing 2024-11-17T15:38:36,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. 2024-11-17T15:38:36,629 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => fdca08c93967309fb9c5dcd33e10e7d5, NAME => 'TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:36,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => fdca08c93967309fb9c5dcd33e10e7d5, NAME => 'TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:36,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fdca08c93967309fb9c5dcd33e10e7d5, NAME => 'TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fdca08c93967309fb9c5dcd33e10e7d5, NAME => 'TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:36,637 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/C/f2ab4b89981a44aaa1ad9c44815c0289, entries=150, sequenceid=133, filesize=11.8 K 2024-11-17T15:38:36,638 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for fdca08c93967309fb9c5dcd33e10e7d5 in 1313ms, sequenceid=133, compaction requested=true 2024-11-17T15:38:36,638 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fdca08c93967309fb9c5dcd33e10e7d5: 2024-11-17T15:38:36,639 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T15:38:36,639 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fdca08c93967309fb9c5dcd33e10e7d5:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T15:38:36,639 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T15:38:36,639 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T15:38:36,639 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fdca08c93967309fb9c5dcd33e10e7d5:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T15:38:36,639 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T15:38:36,639 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fdca08c93967309fb9c5dcd33e10e7d5:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T15:38:36,639 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T15:38:36,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-17T15:38:36,647 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T15:38:36,647 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1541): fdca08c93967309fb9c5dcd33e10e7d5/B is initiating minor compaction (all files) 2024-11-17T15:38:36,647 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fdca08c93967309fb9c5dcd33e10e7d5/B in TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. 2024-11-17T15:38:36,648 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/B/14c24457fdf248e1abefb22cc1722838, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/B/858a9b0419cc444a9f6f9fb54aaf1938, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/B/ced4749f804a4d54b349240bd2592b4c] into tmpdir=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp, totalSize=35.5 K 2024-11-17T15:38:36,648 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101765 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T15:38:36,648 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting 14c24457fdf248e1abefb22cc1722838, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731857910728 2024-11-17T15:38:36,648 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1541): fdca08c93967309fb9c5dcd33e10e7d5/A is initiating minor compaction (all files) 2024-11-17T15:38:36,648 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fdca08c93967309fb9c5dcd33e10e7d5/A in TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. 2024-11-17T15:38:36,649 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/A/06f5c7669a334db7b2425a36a5e0b8d2, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/A/2fae58e5cf364c1aab9154350291067f, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/A/ad6382eeb92b43d282cea63b5c453a4d] into tmpdir=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp, totalSize=99.4 K 2024-11-17T15:38:36,649 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. 2024-11-17T15:38:36,649 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. files: [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/A/06f5c7669a334db7b2425a36a5e0b8d2, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/A/2fae58e5cf364c1aab9154350291067f, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/A/ad6382eeb92b43d282cea63b5c453a4d] 2024-11-17T15:38:36,649 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting 858a9b0419cc444a9f6f9fb54aaf1938, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731857911930 2024-11-17T15:38:36,649 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 06f5c7669a334db7b2425a36a5e0b8d2, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731857910728 2024-11-17T15:38:36,649 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting ced4749f804a4d54b349240bd2592b4c, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731857914154 2024-11-17T15:38:36,650 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2fae58e5cf364c1aab9154350291067f, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731857911930 2024-11-17T15:38:36,650 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting ad6382eeb92b43d282cea63b5c453a4d, keycount=200, bloomtype=ROW, size=38.7 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731857914135 2024-11-17T15:38:36,661 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fdca08c93967309fb9c5dcd33e10e7d5#B#compaction#326 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T15:38:36,662 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/B/d05751d5d1c44113a649265f06ef1f78 is 50, key is test_row_0/B:col10/1731857915324/Put/seqid=0 2024-11-17T15:38:36,665 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=fdca08c93967309fb9c5dcd33e10e7d5] 2024-11-17T15:38:36,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742213_1389 (size=12409) 2024-11-17T15:38:36,681 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241117bd9478649555480cb67cde34e7354cc1_fdca08c93967309fb9c5dcd33e10e7d5 store=[table=TestAcidGuarantees family=A region=fdca08c93967309fb9c5dcd33e10e7d5] 2024-11-17T15:38:36,683 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241117bd9478649555480cb67cde34e7354cc1_fdca08c93967309fb9c5dcd33e10e7d5, store=[table=TestAcidGuarantees family=A region=fdca08c93967309fb9c5dcd33e10e7d5] 2024-11-17T15:38:36,683 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117bd9478649555480cb67cde34e7354cc1_fdca08c93967309fb9c5dcd33e10e7d5 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=fdca08c93967309fb9c5dcd33e10e7d5] 2024-11-17T15:38:36,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742214_1390 (size=4469) 2024-11-17T15:38:36,774 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T15:38:36,774 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fdca08c93967309fb9c5dcd33e10e7d5 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-17T15:38:36,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(8855): Flush requested on fdca08c93967309fb9c5dcd33e10e7d5 2024-11-17T15:38:36,782 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-17T15:38:36,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. 2024-11-17T15:38:36,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. as already flushing 2024-11-17T15:38:36,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. 2024-11-17T15:38:36,783 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => fdca08c93967309fb9c5dcd33e10e7d5, NAME => 'TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:36,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => fdca08c93967309fb9c5dcd33e10e7d5, NAME => 'TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:36,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fdca08c93967309fb9c5dcd33e10e7d5, NAME => 'TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fdca08c93967309fb9c5dcd33e10e7d5, NAME => 'TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:36,807 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117f3daf79a229f4bd7b992b9c2ec427c6e_fdca08c93967309fb9c5dcd33e10e7d5 is 50, key is test_row_0/A:col10/1731857915338/Put/seqid=0 2024-11-17T15:38:36,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742215_1391 (size=14794) 2024-11-17T15:38:36,936 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-17T15:38:36,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. 2024-11-17T15:38:36,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. as already flushing 2024-11-17T15:38:36,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. 2024-11-17T15:38:36,937 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => fdca08c93967309fb9c5dcd33e10e7d5, NAME => 'TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:36,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => fdca08c93967309fb9c5dcd33e10e7d5, NAME => 'TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:36,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fdca08c93967309fb9c5dcd33e10e7d5, NAME => 'TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fdca08c93967309fb9c5dcd33e10e7d5, NAME => 'TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:36,952 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:36,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49652 deadline: 1731857976937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:36,953 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:36,953 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:36,953 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:37,072 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:37,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49652 deadline: 1731857977063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:37,074 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:37,074 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:37,074 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:37,090 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/B/d05751d5d1c44113a649265f06ef1f78 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/B/d05751d5d1c44113a649265f06ef1f78 2024-11-17T15:38:37,090 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-17T15:38:37,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. 2024-11-17T15:38:37,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. as already flushing 2024-11-17T15:38:37,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. 2024-11-17T15:38:37,090 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => fdca08c93967309fb9c5dcd33e10e7d5, NAME => 'TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:37,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => fdca08c93967309fb9c5dcd33e10e7d5, NAME => 'TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:37,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fdca08c93967309fb9c5dcd33e10e7d5, NAME => 'TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fdca08c93967309fb9c5dcd33e10e7d5, NAME => 'TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:37,096 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fdca08c93967309fb9c5dcd33e10e7d5/B of fdca08c93967309fb9c5dcd33e10e7d5 into d05751d5d1c44113a649265f06ef1f78(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T15:38:37,096 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fdca08c93967309fb9c5dcd33e10e7d5: 2024-11-17T15:38:37,096 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., storeName=fdca08c93967309fb9c5dcd33e10e7d5/B, priority=13, startTime=1731857916639; duration=0sec 2024-11-17T15:38:37,096 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T15:38:37,097 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fdca08c93967309fb9c5dcd33e10e7d5:B 2024-11-17T15:38:37,097 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fdca08c93967309fb9c5dcd33e10e7d5#A#compaction#327 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T15:38:37,098 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/A/2a8f89ce907347f281a1f464438b1248 is 175, key is test_row_0/A:col10/1731857915324/Put/seqid=0 2024-11-17T15:38:37,098 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T15:38:37,100 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T15:38:37,100 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1541): fdca08c93967309fb9c5dcd33e10e7d5/C is initiating minor compaction (all files) 2024-11-17T15:38:37,100 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fdca08c93967309fb9c5dcd33e10e7d5/C in TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. 2024-11-17T15:38:37,100 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/C/ffd5f307d98949068959f8e860c04ef2, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/C/b6bbd5aab24c49e9a68e82b9991dc29c, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/C/f2ab4b89981a44aaa1ad9c44815c0289] into tmpdir=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp, totalSize=35.5 K 2024-11-17T15:38:37,101 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting ffd5f307d98949068959f8e860c04ef2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731857910728 2024-11-17T15:38:37,101 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting b6bbd5aab24c49e9a68e82b9991dc29c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731857911930 2024-11-17T15:38:37,101 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting f2ab4b89981a44aaa1ad9c44815c0289, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731857914154 2024-11-17T15:38:37,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742216_1392 (size=31363) 2024-11-17T15:38:37,122 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fdca08c93967309fb9c5dcd33e10e7d5#C#compaction#329 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T15:38:37,123 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/C/1985e629fcd842159ceb2d1a1f5f3b98 is 50, key is test_row_0/C:col10/1731857915324/Put/seqid=0 2024-11-17T15:38:37,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742217_1393 (size=12409) 2024-11-17T15:38:37,131 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/A/2a8f89ce907347f281a1f464438b1248 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/A/2a8f89ce907347f281a1f464438b1248 2024-11-17T15:38:37,134 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/C/1985e629fcd842159ceb2d1a1f5f3b98 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/C/1985e629fcd842159ceb2d1a1f5f3b98 2024-11-17T15:38:37,136 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fdca08c93967309fb9c5dcd33e10e7d5/A of fdca08c93967309fb9c5dcd33e10e7d5 into 2a8f89ce907347f281a1f464438b1248(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T15:38:37,136 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fdca08c93967309fb9c5dcd33e10e7d5: 2024-11-17T15:38:37,136 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., storeName=fdca08c93967309fb9c5dcd33e10e7d5/A, priority=13, startTime=1731857916639; duration=0sec 2024-11-17T15:38:37,136 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T15:38:37,136 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fdca08c93967309fb9c5dcd33e10e7d5:A 2024-11-17T15:38:37,140 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fdca08c93967309fb9c5dcd33e10e7d5/C of fdca08c93967309fb9c5dcd33e10e7d5 into 1985e629fcd842159ceb2d1a1f5f3b98(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T15:38:37,140 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fdca08c93967309fb9c5dcd33e10e7d5: 2024-11-17T15:38:37,140 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., storeName=fdca08c93967309fb9c5dcd33e10e7d5/C, priority=13, startTime=1731857916639; duration=0sec 2024-11-17T15:38:37,140 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T15:38:37,140 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fdca08c93967309fb9c5dcd33e10e7d5:C 2024-11-17T15:38:37,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-17T15:38:37,233 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:37,241 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117f3daf79a229f4bd7b992b9c2ec427c6e_fdca08c93967309fb9c5dcd33e10e7d5 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117f3daf79a229f4bd7b992b9c2ec427c6e_fdca08c93967309fb9c5dcd33e10e7d5 2024-11-17T15:38:37,242 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/A/94a6407329874331add41f4e4ab6a95d, store: [table=TestAcidGuarantees family=A region=fdca08c93967309fb9c5dcd33e10e7d5] 2024-11-17T15:38:37,242 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/A/94a6407329874331add41f4e4ab6a95d is 175, key is test_row_0/A:col10/1731857915338/Put/seqid=0 2024-11-17T15:38:37,244 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-17T15:38:37,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. 2024-11-17T15:38:37,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. as already flushing 2024-11-17T15:38:37,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. 2024-11-17T15:38:37,244 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => fdca08c93967309fb9c5dcd33e10e7d5, NAME => 'TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:37,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => fdca08c93967309fb9c5dcd33e10e7d5, NAME => 'TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:37,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fdca08c93967309fb9c5dcd33e10e7d5, NAME => 'TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fdca08c93967309fb9c5dcd33e10e7d5, NAME => 'TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:37,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742218_1394 (size=39749) 2024-11-17T15:38:37,288 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:37,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49652 deadline: 1731857977283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:37,288 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:37,289 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:37,289 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:37,396 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-17T15:38:37,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. 2024-11-17T15:38:37,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. as already flushing 2024-11-17T15:38:37,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. 2024-11-17T15:38:37,397 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => fdca08c93967309fb9c5dcd33e10e7d5, NAME => 'TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:37,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => fdca08c93967309fb9c5dcd33e10e7d5, NAME => 'TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:37,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fdca08c93967309fb9c5dcd33e10e7d5, NAME => 'TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fdca08c93967309fb9c5dcd33e10e7d5, NAME => 'TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:37,549 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-17T15:38:37,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. 2024-11-17T15:38:37,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. as already flushing 2024-11-17T15:38:37,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. 2024-11-17T15:38:37,550 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => fdca08c93967309fb9c5dcd33e10e7d5, NAME => 'TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:37,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => fdca08c93967309fb9c5dcd33e10e7d5, NAME => 'TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:37,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fdca08c93967309fb9c5dcd33e10e7d5, NAME => 'TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fdca08c93967309fb9c5dcd33e10e7d5, NAME => 'TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:37,552 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:37,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49654 deadline: 1731857977543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:37,553 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 7a780d55532c,32955,1731857800648 for 'test_row_0' in fdca08c93967309fb9c5dcd33e10e7d5 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2185 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T15:38:37,554 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:37,554 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:37,554 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:37,584 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:37,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49690 deadline: 1731857977573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:37,584 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:37,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49700 deadline: 1731857977573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:37,584 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 7a780d55532c,32955,1731857800648 for 'test_row_0' in fdca08c93967309fb9c5dcd33e10e7d5 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2206 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T15:38:37,585 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:37,585 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:37,585 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:37,585 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 7a780d55532c,32955,1731857800648 for 'test_row_1' in fdca08c93967309fb9c5dcd33e10e7d5 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2206 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T15:38:37,585 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:37,585 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:37,585 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:37,592 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:37,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49680 deadline: 1731857977583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:37,593 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 7a780d55532c,32955,1731857800648 for 'test_row_2' in fdca08c93967309fb9c5dcd33e10e7d5 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2212 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T15:38:37,593 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:37,593 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:37,594 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:37,602 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:37,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49652 deadline: 1731857977593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:37,602 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:37,602 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:37,602 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:37,671 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=155, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/A/94a6407329874331add41f4e4ab6a95d 2024-11-17T15:38:37,679 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/B/3571e894a51b4b95a24e450427956fdc is 50, key is test_row_0/B:col10/1731857915338/Put/seqid=0 2024-11-17T15:38:37,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742219_1395 (size=12151) 2024-11-17T15:38:37,703 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/B/3571e894a51b4b95a24e450427956fdc 2024-11-17T15:38:37,707 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-17T15:38:37,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. 2024-11-17T15:38:37,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. as already flushing 2024-11-17T15:38:37,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. 2024-11-17T15:38:37,707 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => fdca08c93967309fb9c5dcd33e10e7d5, NAME => 'TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:37,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => fdca08c93967309fb9c5dcd33e10e7d5, NAME => 'TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:37,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fdca08c93967309fb9c5dcd33e10e7d5, NAME => 'TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fdca08c93967309fb9c5dcd33e10e7d5, NAME => 'TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:37,715 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/C/e7ab5e6acb984825836e2600e542f4eb is 50, key is test_row_0/C:col10/1731857915338/Put/seqid=0 2024-11-17T15:38:37,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742220_1396 (size=12151) 2024-11-17T15:38:37,861 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-17T15:38:37,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. 2024-11-17T15:38:37,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. as already flushing 2024-11-17T15:38:37,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. 2024-11-17T15:38:37,861 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => fdca08c93967309fb9c5dcd33e10e7d5, NAME => 'TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:37,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => fdca08c93967309fb9c5dcd33e10e7d5, NAME => 'TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:37,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fdca08c93967309fb9c5dcd33e10e7d5, NAME => 'TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fdca08c93967309fb9c5dcd33e10e7d5, NAME => 'TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:38,013 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-17T15:38:38,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. 2024-11-17T15:38:38,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. as already flushing 2024-11-17T15:38:38,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. 2024-11-17T15:38:38,014 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => fdca08c93967309fb9c5dcd33e10e7d5, NAME => 'TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:38,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => fdca08c93967309fb9c5dcd33e10e7d5, NAME => 'TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:38,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fdca08c93967309fb9c5dcd33e10e7d5, NAME => 'TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fdca08c93967309fb9c5dcd33e10e7d5, NAME => 'TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:38,118 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:38,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49652 deadline: 1731857978112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:38,118 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:38,118 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:38,118 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:38,133 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/C/e7ab5e6acb984825836e2600e542f4eb 2024-11-17T15:38:38,137 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/A/94a6407329874331add41f4e4ab6a95d as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/A/94a6407329874331add41f4e4ab6a95d 2024-11-17T15:38:38,141 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/A/94a6407329874331add41f4e4ab6a95d, entries=200, sequenceid=155, filesize=38.8 K 2024-11-17T15:38:38,141 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/B/3571e894a51b4b95a24e450427956fdc as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/B/3571e894a51b4b95a24e450427956fdc 2024-11-17T15:38:38,145 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/B/3571e894a51b4b95a24e450427956fdc, entries=150, sequenceid=155, filesize=11.9 K 2024-11-17T15:38:38,145 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/C/e7ab5e6acb984825836e2600e542f4eb as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/C/e7ab5e6acb984825836e2600e542f4eb 2024-11-17T15:38:38,149 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/C/e7ab5e6acb984825836e2600e542f4eb, entries=150, sequenceid=155, filesize=11.9 K 2024-11-17T15:38:38,150 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for fdca08c93967309fb9c5dcd33e10e7d5 in 1376ms, sequenceid=155, compaction requested=false 2024-11-17T15:38:38,150 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fdca08c93967309fb9c5dcd33e10e7d5: 2024-11-17T15:38:38,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-17T15:38:38,166 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-17T15:38:38,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. 2024-11-17T15:38:38,166 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2902): Flushing fdca08c93967309fb9c5dcd33e10e7d5 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-17T15:38:38,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411174f5f6e9ba339407ba367fa5217c806af_fdca08c93967309fb9c5dcd33e10e7d5 is 50, key is test_row_0/A:col10/1731857916891/Put/seqid=0 2024-11-17T15:38:38,177 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742221_1397 (size=12304) 2024-11-17T15:38:38,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:38,583 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411174f5f6e9ba339407ba367fa5217c806af_fdca08c93967309fb9c5dcd33e10e7d5 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411174f5f6e9ba339407ba367fa5217c806af_fdca08c93967309fb9c5dcd33e10e7d5 2024-11-17T15:38:38,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/A/b6aa2b9c401648c7ae33a5dde6844525, store: [table=TestAcidGuarantees family=A region=fdca08c93967309fb9c5dcd33e10e7d5] 2024-11-17T15:38:38,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/A/b6aa2b9c401648c7ae33a5dde6844525 is 175, key is test_row_0/A:col10/1731857916891/Put/seqid=0 2024-11-17T15:38:38,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742222_1398 (size=31105) 2024-11-17T15:38:38,997 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=172, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/A/b6aa2b9c401648c7ae33a5dde6844525 2024-11-17T15:38:39,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/B/2737368a0e504af496a6a2c56bc193ab is 50, key is test_row_0/B:col10/1731857916891/Put/seqid=0 2024-11-17T15:38:39,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742223_1399 (size=12151) 2024-11-17T15:38:39,028 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/B/2737368a0e504af496a6a2c56bc193ab 2024-11-17T15:38:39,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/C/856a3d4a5cfa4390af643af0d59c95dd is 50, key is test_row_0/C:col10/1731857916891/Put/seqid=0 2024-11-17T15:38:39,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742224_1400 (size=12151) 2024-11-17T15:38:39,059 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/C/856a3d4a5cfa4390af643af0d59c95dd 2024-11-17T15:38:39,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/A/b6aa2b9c401648c7ae33a5dde6844525 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/A/b6aa2b9c401648c7ae33a5dde6844525 2024-11-17T15:38:39,074 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/A/b6aa2b9c401648c7ae33a5dde6844525, entries=150, sequenceid=172, filesize=30.4 K 2024-11-17T15:38:39,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/B/2737368a0e504af496a6a2c56bc193ab as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/B/2737368a0e504af496a6a2c56bc193ab 2024-11-17T15:38:39,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,080 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/B/2737368a0e504af496a6a2c56bc193ab, entries=150, sequenceid=172, filesize=11.9 K 2024-11-17T15:38:39,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/C/856a3d4a5cfa4390af643af0d59c95dd as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/C/856a3d4a5cfa4390af643af0d59c95dd 2024-11-17T15:38:39,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,085 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/C/856a3d4a5cfa4390af643af0d59c95dd, entries=150, sequenceid=172, filesize=11.9 K 2024-11-17T15:38:39,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,086 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=0 B/0 for fdca08c93967309fb9c5dcd33e10e7d5 in 920ms, sequenceid=172, compaction requested=true 2024-11-17T15:38:39,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2603): Flush status journal for fdca08c93967309fb9c5dcd33e10e7d5: 2024-11-17T15:38:39,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. 2024-11-17T15:38:39,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=110 2024-11-17T15:38:39,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.HMaster(4169): Remote procedure done, pid=110 2024-11-17T15:38:39,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,090 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-11-17T15:38:39,090 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.0710 sec 2024-11-17T15:38:39,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,091 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-17T15:38:39,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,092 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees in 3.0770 sec 2024-11-17T15:38:39,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(8855): Flush requested on fdca08c93967309fb9c5dcd33e10e7d5 2024-11-17T15:38:39,630 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T15:38:39,630 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fdca08c93967309fb9c5dcd33e10e7d5 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-17T15:38:39,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,662 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411172c704482f45a43d5b639ab13d6cbd64b_fdca08c93967309fb9c5dcd33e10e7d5 is 50, key is test_row_0/A:col10/1731857919629/Put/seqid=0 2024-11-17T15:38:39,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:39,702 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742225_1401 (size=14794) 2024-11-17T15:38:39,757 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:39,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49680 deadline: 1731857979742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:39,760 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:39,760 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:39,760 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:39,760 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:39,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49652 deadline: 1731857979746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:39,761 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:39,761 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:39,761 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:39,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49654 deadline: 1731857979749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:39,761 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:39,761 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:39,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49700 deadline: 1731857979749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:39,761 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:39,761 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:39,761 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:39,761 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:39,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49690 deadline: 1731857979751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:39,762 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:39,762 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:39,762 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:39,762 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:39,762 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:39,762 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:39,867 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:39,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49680 deadline: 1731857979863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:39,868 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:39,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49652 deadline: 1731857979863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:39,868 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:39,868 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:39,868 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:39,868 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:39,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49700 deadline: 1731857979864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:39,868 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:39,868 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:39,868 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:39,868 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:39,868 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:39,868 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:39,881 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:39,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49690 deadline: 1731857979873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:39,882 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:39,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49654 deadline: 1731857979873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:39,882 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:39,882 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:39,882 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:39,883 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:39,883 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:39,883 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:40,085 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:40,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49700 deadline: 1731857980074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:40,086 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:40,086 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:40,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49652 deadline: 1731857980074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:40,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49680 deadline: 1731857980074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:40,086 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:40,086 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:40,086 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:40,086 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:40,086 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:40,086 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:40,087 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:40,087 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:40,087 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:40,103 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:40,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49654 deadline: 1731857980093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:40,104 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:40,104 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:40,104 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:40,104 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:40,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49690 deadline: 1731857980093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:40,104 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:40,104 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:40,104 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:40,104 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:40,112 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411172c704482f45a43d5b639ab13d6cbd64b_fdca08c93967309fb9c5dcd33e10e7d5 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411172c704482f45a43d5b639ab13d6cbd64b_fdca08c93967309fb9c5dcd33e10e7d5 2024-11-17T15:38:40,113 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/A/a999ee2f4a1d479688b3fee212ef6c65, store: [table=TestAcidGuarantees family=A region=fdca08c93967309fb9c5dcd33e10e7d5] 2024-11-17T15:38:40,114 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/A/a999ee2f4a1d479688b3fee212ef6c65 is 175, key is test_row_0/A:col10/1731857919629/Put/seqid=0 2024-11-17T15:38:40,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742226_1402 (size=39745) 2024-11-17T15:38:40,127 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=183, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/A/a999ee2f4a1d479688b3fee212ef6c65 2024-11-17T15:38:40,139 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/B/f742ccdf57f14934a865baa88511d959 is 50, key is test_row_0/B:col10/1731857919629/Put/seqid=0 2024-11-17T15:38:40,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742227_1403 (size=9757) 2024-11-17T15:38:40,159 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=183 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/B/f742ccdf57f14934a865baa88511d959 2024-11-17T15:38:40,172 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/C/7c832cf87da1477fbba1671ba4d74b5d is 50, key is test_row_0/C:col10/1731857919629/Put/seqid=0 2024-11-17T15:38:40,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-17T15:38:40,183 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T15:38:40,183 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T15:38:40,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees 2024-11-17T15:38:40,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-17T15:38:40,186 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T15:38:40,194 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T15:38:40,194 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T15:38:40,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742228_1404 (size=9757) 2024-11-17T15:38:40,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-17T15:38:40,347 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-17T15:38:40,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. 2024-11-17T15:38:40,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. as already flushing 2024-11-17T15:38:40,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. 2024-11-17T15:38:40,348 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => fdca08c93967309fb9c5dcd33e10e7d5, NAME => 'TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:40,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => fdca08c93967309fb9c5dcd33e10e7d5, NAME => 'TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:40,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fdca08c93967309fb9c5dcd33e10e7d5, NAME => 'TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fdca08c93967309fb9c5dcd33e10e7d5, NAME => 'TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:40,409 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:40,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49652 deadline: 1731857980393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:40,409 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:40,410 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:40,410 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:40,410 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:40,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49700 deadline: 1731857980393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:40,410 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:40,410 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:40,410 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:40,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49680 deadline: 1731857980393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:40,410 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:40,410 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:40,410 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:40,410 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:40,429 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:40,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49654 deadline: 1731857980415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:40,430 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:40,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49690 deadline: 1731857980416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:40,430 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:40,430 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:40,430 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:40,430 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:40,430 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:40,430 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:40,501 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-17T15:38:40,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. 2024-11-17T15:38:40,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. as already flushing 2024-11-17T15:38:40,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. 2024-11-17T15:38:40,501 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => fdca08c93967309fb9c5dcd33e10e7d5, NAME => 'TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:40,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => fdca08c93967309fb9c5dcd33e10e7d5, NAME => 'TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:40,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fdca08c93967309fb9c5dcd33e10e7d5, NAME => 'TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fdca08c93967309fb9c5dcd33e10e7d5, NAME => 'TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:40,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-17T15:38:40,612 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=183 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/C/7c832cf87da1477fbba1671ba4d74b5d 2024-11-17T15:38:40,618 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/A/a999ee2f4a1d479688b3fee212ef6c65 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/A/a999ee2f4a1d479688b3fee212ef6c65 2024-11-17T15:38:40,625 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/A/a999ee2f4a1d479688b3fee212ef6c65, entries=200, sequenceid=183, filesize=38.8 K 2024-11-17T15:38:40,626 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/B/f742ccdf57f14934a865baa88511d959 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/B/f742ccdf57f14934a865baa88511d959 2024-11-17T15:38:40,633 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/B/f742ccdf57f14934a865baa88511d959, entries=100, sequenceid=183, filesize=9.5 K 2024-11-17T15:38:40,634 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/C/7c832cf87da1477fbba1671ba4d74b5d as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/C/7c832cf87da1477fbba1671ba4d74b5d 2024-11-17T15:38:40,639 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/C/7c832cf87da1477fbba1671ba4d74b5d, entries=100, sequenceid=183, filesize=9.5 K 2024-11-17T15:38:40,640 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for fdca08c93967309fb9c5dcd33e10e7d5 in 1010ms, sequenceid=183, compaction requested=true 2024-11-17T15:38:40,640 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fdca08c93967309fb9c5dcd33e10e7d5: 2024-11-17T15:38:40,640 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T15:38:40,641 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fdca08c93967309fb9c5dcd33e10e7d5:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T15:38:40,641 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T15:38:40,641 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fdca08c93967309fb9c5dcd33e10e7d5:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T15:38:40,641 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T15:38:40,641 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fdca08c93967309fb9c5dcd33e10e7d5:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T15:38:40,641 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-17T15:38:40,642 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 141962 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T15:38:40,642 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1541): fdca08c93967309fb9c5dcd33e10e7d5/A is initiating minor compaction (all files) 2024-11-17T15:38:40,642 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fdca08c93967309fb9c5dcd33e10e7d5/A in TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. 2024-11-17T15:38:40,642 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/A/2a8f89ce907347f281a1f464438b1248, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/A/94a6407329874331add41f4e4ab6a95d, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/A/b6aa2b9c401648c7ae33a5dde6844525, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/A/a999ee2f4a1d479688b3fee212ef6c65] into tmpdir=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp, totalSize=138.6 K 2024-11-17T15:38:40,642 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. 2024-11-17T15:38:40,642 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. files: [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/A/2a8f89ce907347f281a1f464438b1248, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/A/94a6407329874331add41f4e4ab6a95d, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/A/b6aa2b9c401648c7ae33a5dde6844525, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/A/a999ee2f4a1d479688b3fee212ef6c65] 2024-11-17T15:38:40,643 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting 2a8f89ce907347f281a1f464438b1248, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731857914154 2024-11-17T15:38:40,643 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T15:38:40,643 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting 94a6407329874331add41f4e4ab6a95d, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731857915338 2024-11-17T15:38:40,643 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting b6aa2b9c401648c7ae33a5dde6844525, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731857916891 2024-11-17T15:38:40,644 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting a999ee2f4a1d479688b3fee212ef6c65, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1731857919535 2024-11-17T15:38:40,645 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46468 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T15:38:40,645 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1541): fdca08c93967309fb9c5dcd33e10e7d5/B is initiating minor compaction (all files) 2024-11-17T15:38:40,645 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fdca08c93967309fb9c5dcd33e10e7d5/B in TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. 2024-11-17T15:38:40,645 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/B/d05751d5d1c44113a649265f06ef1f78, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/B/3571e894a51b4b95a24e450427956fdc, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/B/2737368a0e504af496a6a2c56bc193ab, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/B/f742ccdf57f14934a865baa88511d959] into tmpdir=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp, totalSize=45.4 K 2024-11-17T15:38:40,648 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting d05751d5d1c44113a649265f06ef1f78, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731857914154 2024-11-17T15:38:40,648 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3571e894a51b4b95a24e450427956fdc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731857915338 2024-11-17T15:38:40,649 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2737368a0e504af496a6a2c56bc193ab, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731857916891 2024-11-17T15:38:40,653 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting f742ccdf57f14934a865baa88511d959, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1731857919535 2024-11-17T15:38:40,655 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-17T15:38:40,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. 2024-11-17T15:38:40,655 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2902): Flushing fdca08c93967309fb9c5dcd33e10e7d5 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-17T15:38:40,662 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=fdca08c93967309fb9c5dcd33e10e7d5] 2024-11-17T15:38:40,666 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fdca08c93967309fb9c5dcd33e10e7d5#B#compaction#339 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T15:38:40,667 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/B/7214dc60e22340c8a33ba09a2c775ba0 is 50, key is test_row_0/B:col10/1731857919629/Put/seqid=0 2024-11-17T15:38:40,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117b2263032a0a849ceb61e72f87c69c54a_fdca08c93967309fb9c5dcd33e10e7d5 is 50, key is test_row_0/A:col10/1731857919721/Put/seqid=0 2024-11-17T15:38:40,685 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241117d4046d82bbb04602ae980ab27f82f177_fdca08c93967309fb9c5dcd33e10e7d5 store=[table=TestAcidGuarantees family=A region=fdca08c93967309fb9c5dcd33e10e7d5] 2024-11-17T15:38:40,687 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241117d4046d82bbb04602ae980ab27f82f177_fdca08c93967309fb9c5dcd33e10e7d5, store=[table=TestAcidGuarantees family=A region=fdca08c93967309fb9c5dcd33e10e7d5] 2024-11-17T15:38:40,688 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117d4046d82bbb04602ae980ab27f82f177_fdca08c93967309fb9c5dcd33e10e7d5 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=fdca08c93967309fb9c5dcd33e10e7d5] 2024-11-17T15:38:40,704 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742229_1405 (size=12595) 2024-11-17T15:38:40,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742230_1406 (size=12304) 2024-11-17T15:38:40,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742231_1407 (size=4469) 2024-11-17T15:38:40,711 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fdca08c93967309fb9c5dcd33e10e7d5#A#compaction#338 average throughput is 0.51 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T15:38:40,712 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/A/f4cd3f0ce3f74b5483bf2d7e2ca41145 is 175, key is test_row_0/A:col10/1731857919629/Put/seqid=0 2024-11-17T15:38:40,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742232_1408 (size=31656) 2024-11-17T15:38:40,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-17T15:38:40,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(8855): Flush requested on fdca08c93967309fb9c5dcd33e10e7d5 2024-11-17T15:38:40,915 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. as already flushing 2024-11-17T15:38:40,941 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:40,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49652 deadline: 1731857980930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:40,941 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:40,941 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:40,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49680 deadline: 1731857980931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:40,942 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:40,942 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:40,942 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:40,942 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:40,942 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:40,943 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:40,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49654 deadline: 1731857980932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:40,944 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:40,944 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:40,944 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:40,954 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:40,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49700 deadline: 1731857980938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:40,955 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:40,955 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:40,955 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:40,956 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:40,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49690 deadline: 1731857980943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:40,956 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:40,956 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:40,956 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:41,058 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:41,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49652 deadline: 1731857981053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:41,059 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:41,059 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:41,059 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:41,067 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:41,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49680 deadline: 1731857981053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:41,067 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:41,067 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:41,067 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:41,072 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:41,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49700 deadline: 1731857981063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:41,073 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:41,073 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:41,073 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:41,109 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/B/7214dc60e22340c8a33ba09a2c775ba0 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/B/7214dc60e22340c8a33ba09a2c775ba0 2024-11-17T15:38:41,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:41,113 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117b2263032a0a849ceb61e72f87c69c54a_fdca08c93967309fb9c5dcd33e10e7d5 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117b2263032a0a849ceb61e72f87c69c54a_fdca08c93967309fb9c5dcd33e10e7d5 2024-11-17T15:38:41,113 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in fdca08c93967309fb9c5dcd33e10e7d5/B of fdca08c93967309fb9c5dcd33e10e7d5 into 7214dc60e22340c8a33ba09a2c775ba0(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T15:38:41,113 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fdca08c93967309fb9c5dcd33e10e7d5: 2024-11-17T15:38:41,113 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., storeName=fdca08c93967309fb9c5dcd33e10e7d5/B, priority=12, startTime=1731857920641; duration=0sec 2024-11-17T15:38:41,113 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T15:38:41,113 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fdca08c93967309fb9c5dcd33e10e7d5:B 2024-11-17T15:38:41,113 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T15:38:41,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/A/f36338c0bab649a5a44aba1189f1c29f, store: [table=TestAcidGuarantees family=A region=fdca08c93967309fb9c5dcd33e10e7d5] 2024-11-17T15:38:41,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/A/f36338c0bab649a5a44aba1189f1c29f is 175, key is test_row_0/A:col10/1731857919721/Put/seqid=0 2024-11-17T15:38:41,125 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/A/f4cd3f0ce3f74b5483bf2d7e2ca41145 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/A/f4cd3f0ce3f74b5483bf2d7e2ca41145 2024-11-17T15:38:41,130 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in fdca08c93967309fb9c5dcd33e10e7d5/A of fdca08c93967309fb9c5dcd33e10e7d5 into f4cd3f0ce3f74b5483bf2d7e2ca41145(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T15:38:41,130 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fdca08c93967309fb9c5dcd33e10e7d5: 2024-11-17T15:38:41,130 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., storeName=fdca08c93967309fb9c5dcd33e10e7d5/A, priority=12, startTime=1731857920640; duration=0sec 2024-11-17T15:38:41,130 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T15:38:41,130 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fdca08c93967309fb9c5dcd33e10e7d5:A 2024-11-17T15:38:41,132 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46468 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T15:38:41,132 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1541): fdca08c93967309fb9c5dcd33e10e7d5/C is initiating minor compaction (all files) 2024-11-17T15:38:41,132 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fdca08c93967309fb9c5dcd33e10e7d5/C in TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. 2024-11-17T15:38:41,133 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/C/1985e629fcd842159ceb2d1a1f5f3b98, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/C/e7ab5e6acb984825836e2600e542f4eb, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/C/856a3d4a5cfa4390af643af0d59c95dd, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/C/7c832cf87da1477fbba1671ba4d74b5d] into tmpdir=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp, totalSize=45.4 K 2024-11-17T15:38:41,133 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1985e629fcd842159ceb2d1a1f5f3b98, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731857914154 2024-11-17T15:38:41,133 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting e7ab5e6acb984825836e2600e542f4eb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731857915338 2024-11-17T15:38:41,134 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 856a3d4a5cfa4390af643af0d59c95dd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731857916891 2024-11-17T15:38:41,136 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7c832cf87da1477fbba1671ba4d74b5d, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1731857919535 2024-11-17T15:38:41,152 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fdca08c93967309fb9c5dcd33e10e7d5#C#compaction#341 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T15:38:41,152 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/C/12b2b5fe96704220a0fd9d763db8313d is 50, key is test_row_0/C:col10/1731857919629/Put/seqid=0 2024-11-17T15:38:41,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742233_1409 (size=31105) 2024-11-17T15:38:41,155 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=209, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/A/f36338c0bab649a5a44aba1189f1c29f 2024-11-17T15:38:41,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742234_1410 (size=12595) 2024-11-17T15:38:41,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/B/515bd0daef4f4bd3a3dece8c33f0f44b is 50, key is test_row_0/B:col10/1731857919721/Put/seqid=0 2024-11-17T15:38:41,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742235_1411 (size=12151) 2024-11-17T15:38:41,273 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:41,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49652 deadline: 1731857981263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:41,274 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:41,274 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:41,274 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:41,283 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:41,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49680 deadline: 1731857981274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:41,283 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:41,283 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:41,283 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:41,291 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:41,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49700 deadline: 1731857981282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:41,292 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:41,292 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:41,292 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:41,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-17T15:38:41,574 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/C/12b2b5fe96704220a0fd9d763db8313d as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/C/12b2b5fe96704220a0fd9d763db8313d 2024-11-17T15:38:41,583 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in fdca08c93967309fb9c5dcd33e10e7d5/C of fdca08c93967309fb9c5dcd33e10e7d5 into 12b2b5fe96704220a0fd9d763db8313d(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T15:38:41,583 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/B/515bd0daef4f4bd3a3dece8c33f0f44b 2024-11-17T15:38:41,583 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fdca08c93967309fb9c5dcd33e10e7d5: 2024-11-17T15:38:41,583 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., storeName=fdca08c93967309fb9c5dcd33e10e7d5/C, priority=12, startTime=1731857920641; duration=0sec 2024-11-17T15:38:41,583 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T15:38:41,585 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fdca08c93967309fb9c5dcd33e10e7d5:C 2024-11-17T15:38:41,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/C/0302936a8a8f43098144e453aec70a0c is 50, key is test_row_0/C:col10/1731857919721/Put/seqid=0 2024-11-17T15:38:41,600 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:41,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49652 deadline: 1731857981583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:41,602 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:41,603 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:41,603 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:41,603 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:41,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49680 deadline: 1731857981593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:41,604 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:41,604 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:41,604 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:41,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742236_1412 (size=12151) 2024-11-17T15:38:41,615 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:41,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49700 deadline: 1731857981603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:41,616 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:41,616 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:41,616 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:41,961 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:41,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49654 deadline: 1731857981953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:41,961 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 7a780d55532c,32955,1731857800648 for 'test_row_2' in fdca08c93967309fb9c5dcd33e10e7d5 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2212 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T15:38:41,962 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:41,962 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:41,962 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:41,971 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:41,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49690 deadline: 1731857981962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:41,971 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 7a780d55532c,32955,1731857800648 for 'test_row_0' in fdca08c93967309fb9c5dcd33e10e7d5 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2220 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T15:38:41,971 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:41,971 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:41,971 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:42,015 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/C/0302936a8a8f43098144e453aec70a0c 2024-11-17T15:38:42,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/A/f36338c0bab649a5a44aba1189f1c29f as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/A/f36338c0bab649a5a44aba1189f1c29f 2024-11-17T15:38:42,023 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/A/f36338c0bab649a5a44aba1189f1c29f, entries=150, sequenceid=209, filesize=30.4 K 2024-11-17T15:38:42,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/B/515bd0daef4f4bd3a3dece8c33f0f44b as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/B/515bd0daef4f4bd3a3dece8c33f0f44b 2024-11-17T15:38:42,028 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/B/515bd0daef4f4bd3a3dece8c33f0f44b, entries=150, sequenceid=209, filesize=11.9 K 2024-11-17T15:38:42,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/C/0302936a8a8f43098144e453aec70a0c as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/C/0302936a8a8f43098144e453aec70a0c 2024-11-17T15:38:42,035 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/C/0302936a8a8f43098144e453aec70a0c, entries=150, sequenceid=209, filesize=11.9 K 2024-11-17T15:38:42,037 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for fdca08c93967309fb9c5dcd33e10e7d5 in 1382ms, sequenceid=209, compaction requested=false 2024-11-17T15:38:42,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2603): Flush status journal for fdca08c93967309fb9c5dcd33e10e7d5: 2024-11-17T15:38:42,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. 2024-11-17T15:38:42,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=112 2024-11-17T15:38:42,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.HMaster(4169): Remote procedure done, pid=112 2024-11-17T15:38:42,041 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-11-17T15:38:42,042 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8460 sec 2024-11-17T15:38:42,044 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees in 1.8580 sec 2024-11-17T15:38:42,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(8855): Flush requested on fdca08c93967309fb9c5dcd33e10e7d5 2024-11-17T15:38:42,118 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T15:38:42,118 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fdca08c93967309fb9c5dcd33e10e7d5 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-17T15:38:42,127 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117116b1019859a44448f1d1624c08ac280_fdca08c93967309fb9c5dcd33e10e7d5 is 50, key is test_row_0/A:col10/1731857922118/Put/seqid=0 2024-11-17T15:38:42,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742237_1413 (size=17284) 2024-11-17T15:38:42,215 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:42,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49652 deadline: 1731857982202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:42,215 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:42,215 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:42,215 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:42,217 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:42,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49700 deadline: 1731857982209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:42,217 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:42,217 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:42,217 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:42,224 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:42,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49680 deadline: 1731857982212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:42,228 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:42,228 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:42,228 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:42,331 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:42,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49700 deadline: 1731857982323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:42,331 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:42,331 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:42,331 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:42,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-17T15:38:42,334 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T15:38:42,334 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T15:38:42,334 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:42,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49652 deadline: 1731857982323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:42,335 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:42,335 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:42,335 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:42,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] procedure2.ProcedureExecutor(1139): Stored pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees 2024-11-17T15:38:42,336 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T15:38:42,337 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T15:38:42,337 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T15:38:42,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-17T15:38:42,344 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:42,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49680 deadline: 1731857982333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:42,346 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:42,346 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:42,346 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:42,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-17T15:38:42,490 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-17T15:38:42,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. 2024-11-17T15:38:42,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. as already flushing 2024-11-17T15:38:42,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. 2024-11-17T15:38:42,490 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => fdca08c93967309fb9c5dcd33e10e7d5, NAME => 'TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:42,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => fdca08c93967309fb9c5dcd33e10e7d5, NAME => 'TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:42,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fdca08c93967309fb9c5dcd33e10e7d5, NAME => 'TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fdca08c93967309fb9c5dcd33e10e7d5, NAME => 'TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:42,535 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:42,540 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117116b1019859a44448f1d1624c08ac280_fdca08c93967309fb9c5dcd33e10e7d5 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117116b1019859a44448f1d1624c08ac280_fdca08c93967309fb9c5dcd33e10e7d5 2024-11-17T15:38:42,542 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/A/c015e8481c0f480097986019990d3c16, store: [table=TestAcidGuarantees family=A region=fdca08c93967309fb9c5dcd33e10e7d5] 2024-11-17T15:38:42,543 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/A/c015e8481c0f480097986019990d3c16 is 175, key is test_row_0/A:col10/1731857922118/Put/seqid=0 2024-11-17T15:38:42,551 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742238_1414 (size=48389) 2024-11-17T15:38:42,554 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:42,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49652 deadline: 1731857982543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:42,555 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:42,555 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:42,555 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:42,555 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:42,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49700 deadline: 1731857982543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:42,556 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:42,556 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:42,556 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:42,560 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:42,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49680 deadline: 1731857982553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:42,560 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:42,560 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:42,560 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:42,647 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-17T15:38:42,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. 2024-11-17T15:38:42,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. as already flushing 2024-11-17T15:38:42,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. 2024-11-17T15:38:42,648 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => fdca08c93967309fb9c5dcd33e10e7d5, NAME => 'TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:42,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => fdca08c93967309fb9c5dcd33e10e7d5, NAME => 'TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:42,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fdca08c93967309fb9c5dcd33e10e7d5, NAME => 'TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fdca08c93967309fb9c5dcd33e10e7d5, NAME => 'TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:42,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-17T15:38:42,803 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-17T15:38:42,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. 2024-11-17T15:38:42,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. as already flushing 2024-11-17T15:38:42,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. 2024-11-17T15:38:42,803 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => fdca08c93967309fb9c5dcd33e10e7d5, NAME => 'TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:42,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => fdca08c93967309fb9c5dcd33e10e7d5, NAME => 'TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:42,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fdca08c93967309fb9c5dcd33e10e7d5, NAME => 'TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fdca08c93967309fb9c5dcd33e10e7d5, NAME => 'TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:42,871 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:42,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49652 deadline: 1731857982863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:42,872 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:42,872 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:42,872 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:42,872 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:42,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49700 deadline: 1731857982863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:42,873 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:42,873 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:42,873 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:42,878 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:42,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49680 deadline: 1731857982873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:42,879 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:42,879 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:42,879 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:42,951 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=224, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/A/c015e8481c0f480097986019990d3c16 2024-11-17T15:38:42,956 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-17T15:38:42,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. 2024-11-17T15:38:42,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. as already flushing 2024-11-17T15:38:42,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. 2024-11-17T15:38:42,957 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => fdca08c93967309fb9c5dcd33e10e7d5, NAME => 'TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:42,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => fdca08c93967309fb9c5dcd33e10e7d5, NAME => 'TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:42,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fdca08c93967309fb9c5dcd33e10e7d5, NAME => 'TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fdca08c93967309fb9c5dcd33e10e7d5, NAME => 'TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:42,962 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/B/beef8fa34b8b4649a1b7bfe875714457 is 50, key is test_row_0/B:col10/1731857922118/Put/seqid=0 2024-11-17T15:38:42,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-17T15:38:42,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742239_1415 (size=12151) 2024-11-17T15:38:42,970 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=224 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/B/beef8fa34b8b4649a1b7bfe875714457 2024-11-17T15:38:42,976 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/C/86e57b9113084e9da7c4b9fba8572cd1 is 50, key is test_row_0/C:col10/1731857922118/Put/seqid=0 2024-11-17T15:38:42,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742240_1416 (size=12151) 2024-11-17T15:38:43,110 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-17T15:38:43,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. 2024-11-17T15:38:43,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. as already flushing 2024-11-17T15:38:43,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. 2024-11-17T15:38:43,110 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => fdca08c93967309fb9c5dcd33e10e7d5, NAME => 'TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:43,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => fdca08c93967309fb9c5dcd33e10e7d5, NAME => 'TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:43,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fdca08c93967309fb9c5dcd33e10e7d5, NAME => 'TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fdca08c93967309fb9c5dcd33e10e7d5, NAME => 'TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:43,185 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7a780d55532c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/A/a34332a6f86542279c4af1aa6e4e4dfe, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/A/1e8ab31ceecf4f959dfc79016730963e, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/A/c9029bd4d1a448febeb28f67eadc5494, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/A/070f58aa875e487d8b2605276feac97e, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/A/81c4b313a35c4ec392a6678b7f6e049b, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/A/ea51521fd44745ed8387b009a19f365a, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/A/06f5c7669a334db7b2425a36a5e0b8d2, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/A/2fae58e5cf364c1aab9154350291067f, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/A/ad6382eeb92b43d282cea63b5c453a4d, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/A/2a8f89ce907347f281a1f464438b1248, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/A/94a6407329874331add41f4e4ab6a95d, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/A/b6aa2b9c401648c7ae33a5dde6844525, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/A/a999ee2f4a1d479688b3fee212ef6c65] to archive 2024-11-17T15:38:43,186 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7a780d55532c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-17T15:38:43,188 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7a780d55532c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/A/a34332a6f86542279c4af1aa6e4e4dfe to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/A/a34332a6f86542279c4af1aa6e4e4dfe 2024-11-17T15:38:43,189 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7a780d55532c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/A/1e8ab31ceecf4f959dfc79016730963e to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/A/1e8ab31ceecf4f959dfc79016730963e 2024-11-17T15:38:43,190 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7a780d55532c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/A/c9029bd4d1a448febeb28f67eadc5494 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/A/c9029bd4d1a448febeb28f67eadc5494 2024-11-17T15:38:43,191 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7a780d55532c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/A/070f58aa875e487d8b2605276feac97e to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/A/070f58aa875e487d8b2605276feac97e 2024-11-17T15:38:43,192 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7a780d55532c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/A/81c4b313a35c4ec392a6678b7f6e049b to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/A/81c4b313a35c4ec392a6678b7f6e049b 2024-11-17T15:38:43,193 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7a780d55532c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/A/ea51521fd44745ed8387b009a19f365a to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/A/ea51521fd44745ed8387b009a19f365a 2024-11-17T15:38:43,194 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7a780d55532c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/A/06f5c7669a334db7b2425a36a5e0b8d2 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/A/06f5c7669a334db7b2425a36a5e0b8d2 2024-11-17T15:38:43,195 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7a780d55532c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/A/2fae58e5cf364c1aab9154350291067f to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/A/2fae58e5cf364c1aab9154350291067f 2024-11-17T15:38:43,197 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7a780d55532c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/A/ad6382eeb92b43d282cea63b5c453a4d to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/A/ad6382eeb92b43d282cea63b5c453a4d 2024-11-17T15:38:43,198 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7a780d55532c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/A/2a8f89ce907347f281a1f464438b1248 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/A/2a8f89ce907347f281a1f464438b1248 2024-11-17T15:38:43,198 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7a780d55532c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/A/94a6407329874331add41f4e4ab6a95d to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/A/94a6407329874331add41f4e4ab6a95d 2024-11-17T15:38:43,200 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7a780d55532c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/A/b6aa2b9c401648c7ae33a5dde6844525 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/A/b6aa2b9c401648c7ae33a5dde6844525 2024-11-17T15:38:43,202 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7a780d55532c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/A/a999ee2f4a1d479688b3fee212ef6c65 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/A/a999ee2f4a1d479688b3fee212ef6c65 2024-11-17T15:38:43,205 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7a780d55532c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/B/64ef4831062f49deb246b5f5662cdd79, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/B/2256f63d3a7b47dda6900cd2783f893a, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/B/36262a5388854577b287964bcfb2c329, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/B/173682b39e1d4aa5b74b9bd7c9b83cc1, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/B/ab3d23baf6884f65956c9dddcd3ecff7, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/B/14c24457fdf248e1abefb22cc1722838, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/B/ed776f37a2e24f4fa54460acfcb1ec72, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/B/858a9b0419cc444a9f6f9fb54aaf1938, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/B/d05751d5d1c44113a649265f06ef1f78, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/B/ced4749f804a4d54b349240bd2592b4c, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/B/3571e894a51b4b95a24e450427956fdc, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/B/2737368a0e504af496a6a2c56bc193ab, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/B/f742ccdf57f14934a865baa88511d959] to archive 2024-11-17T15:38:43,207 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7a780d55532c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-17T15:38:43,209 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7a780d55532c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/B/64ef4831062f49deb246b5f5662cdd79 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/B/64ef4831062f49deb246b5f5662cdd79 2024-11-17T15:38:43,210 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7a780d55532c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/B/2256f63d3a7b47dda6900cd2783f893a to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/B/2256f63d3a7b47dda6900cd2783f893a 2024-11-17T15:38:43,211 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7a780d55532c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/B/36262a5388854577b287964bcfb2c329 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/B/36262a5388854577b287964bcfb2c329 2024-11-17T15:38:43,212 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7a780d55532c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/B/173682b39e1d4aa5b74b9bd7c9b83cc1 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/B/173682b39e1d4aa5b74b9bd7c9b83cc1 2024-11-17T15:38:43,213 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7a780d55532c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/B/ab3d23baf6884f65956c9dddcd3ecff7 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/B/ab3d23baf6884f65956c9dddcd3ecff7 2024-11-17T15:38:43,214 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7a780d55532c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/B/14c24457fdf248e1abefb22cc1722838 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/B/14c24457fdf248e1abefb22cc1722838 2024-11-17T15:38:43,216 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7a780d55532c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/B/ed776f37a2e24f4fa54460acfcb1ec72 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/B/ed776f37a2e24f4fa54460acfcb1ec72 2024-11-17T15:38:43,217 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7a780d55532c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/B/858a9b0419cc444a9f6f9fb54aaf1938 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/B/858a9b0419cc444a9f6f9fb54aaf1938 2024-11-17T15:38:43,218 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7a780d55532c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/B/d05751d5d1c44113a649265f06ef1f78 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/B/d05751d5d1c44113a649265f06ef1f78 2024-11-17T15:38:43,223 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7a780d55532c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/B/ced4749f804a4d54b349240bd2592b4c to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/B/ced4749f804a4d54b349240bd2592b4c 2024-11-17T15:38:43,224 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7a780d55532c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/B/3571e894a51b4b95a24e450427956fdc to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/B/3571e894a51b4b95a24e450427956fdc 2024-11-17T15:38:43,225 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7a780d55532c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/B/2737368a0e504af496a6a2c56bc193ab to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/B/2737368a0e504af496a6a2c56bc193ab 2024-11-17T15:38:43,225 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7a780d55532c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/B/f742ccdf57f14934a865baa88511d959 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/B/f742ccdf57f14934a865baa88511d959 2024-11-17T15:38:43,228 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7a780d55532c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/C/a261db2ea47f4841b60850d38b4687e9, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/C/46f8fdbd721040fc9fe4f70f89753664, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/C/0e1a47e294724e26bc65376e765d0ecc, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/C/9fe9b74d804c47d3b483e9fb7b172bf3, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/C/a30acc0c07b24206a00d4f2f0ceef2f9, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/C/ffd5f307d98949068959f8e860c04ef2, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/C/6a9cf3c9ed5d4b1abb0437cf3a05aac9, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/C/b6bbd5aab24c49e9a68e82b9991dc29c, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/C/1985e629fcd842159ceb2d1a1f5f3b98, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/C/f2ab4b89981a44aaa1ad9c44815c0289, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/C/e7ab5e6acb984825836e2600e542f4eb, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/C/856a3d4a5cfa4390af643af0d59c95dd, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/C/7c832cf87da1477fbba1671ba4d74b5d] to archive 2024-11-17T15:38:43,229 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7a780d55532c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-17T15:38:43,230 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7a780d55532c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/C/a261db2ea47f4841b60850d38b4687e9 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/C/a261db2ea47f4841b60850d38b4687e9 2024-11-17T15:38:43,232 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7a780d55532c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/C/46f8fdbd721040fc9fe4f70f89753664 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/C/46f8fdbd721040fc9fe4f70f89753664 2024-11-17T15:38:43,233 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7a780d55532c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/C/0e1a47e294724e26bc65376e765d0ecc to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/C/0e1a47e294724e26bc65376e765d0ecc 2024-11-17T15:38:43,234 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7a780d55532c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/C/9fe9b74d804c47d3b483e9fb7b172bf3 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/C/9fe9b74d804c47d3b483e9fb7b172bf3 2024-11-17T15:38:43,235 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7a780d55532c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/C/a30acc0c07b24206a00d4f2f0ceef2f9 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/C/a30acc0c07b24206a00d4f2f0ceef2f9 2024-11-17T15:38:43,236 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7a780d55532c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/C/ffd5f307d98949068959f8e860c04ef2 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/C/ffd5f307d98949068959f8e860c04ef2 2024-11-17T15:38:43,237 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7a780d55532c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/C/6a9cf3c9ed5d4b1abb0437cf3a05aac9 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/C/6a9cf3c9ed5d4b1abb0437cf3a05aac9 2024-11-17T15:38:43,238 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7a780d55532c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/C/b6bbd5aab24c49e9a68e82b9991dc29c to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/C/b6bbd5aab24c49e9a68e82b9991dc29c 2024-11-17T15:38:43,239 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7a780d55532c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/C/1985e629fcd842159ceb2d1a1f5f3b98 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/C/1985e629fcd842159ceb2d1a1f5f3b98 2024-11-17T15:38:43,241 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7a780d55532c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/C/f2ab4b89981a44aaa1ad9c44815c0289 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/C/f2ab4b89981a44aaa1ad9c44815c0289 2024-11-17T15:38:43,242 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7a780d55532c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/C/e7ab5e6acb984825836e2600e542f4eb to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/C/e7ab5e6acb984825836e2600e542f4eb 2024-11-17T15:38:43,243 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7a780d55532c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/C/856a3d4a5cfa4390af643af0d59c95dd to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/C/856a3d4a5cfa4390af643af0d59c95dd 2024-11-17T15:38:43,244 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7a780d55532c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/C/7c832cf87da1477fbba1671ba4d74b5d to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/C/7c832cf87da1477fbba1671ba4d74b5d 2024-11-17T15:38:43,263 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-17T15:38:43,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. 2024-11-17T15:38:43,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. as already flushing 2024-11-17T15:38:43,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. 2024-11-17T15:38:43,263 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => fdca08c93967309fb9c5dcd33e10e7d5, NAME => 'TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:43,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => fdca08c93967309fb9c5dcd33e10e7d5, NAME => 'TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:43,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fdca08c93967309fb9c5dcd33e10e7d5, NAME => 'TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fdca08c93967309fb9c5dcd33e10e7d5, NAME => 'TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:43,385 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=224 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/C/86e57b9113084e9da7c4b9fba8572cd1 2024-11-17T15:38:43,388 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/A/c015e8481c0f480097986019990d3c16 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/A/c015e8481c0f480097986019990d3c16 2024-11-17T15:38:43,392 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/A/c015e8481c0f480097986019990d3c16, entries=250, sequenceid=224, filesize=47.3 K 2024-11-17T15:38:43,392 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/B/beef8fa34b8b4649a1b7bfe875714457 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/B/beef8fa34b8b4649a1b7bfe875714457 2024-11-17T15:38:43,392 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:43,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49652 deadline: 1731857983383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:43,393 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:43,393 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:43,393 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:43,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49700 deadline: 1731857983383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:43,393 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:43,393 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:43,393 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:43,393 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:43,393 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:43,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49680 deadline: 1731857983383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:43,394 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:43,394 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:43,394 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:43,396 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/B/beef8fa34b8b4649a1b7bfe875714457, entries=150, sequenceid=224, filesize=11.9 K 2024-11-17T15:38:43,396 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/C/86e57b9113084e9da7c4b9fba8572cd1 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/C/86e57b9113084e9da7c4b9fba8572cd1 2024-11-17T15:38:43,399 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/C/86e57b9113084e9da7c4b9fba8572cd1, entries=150, sequenceid=224, filesize=11.9 K 2024-11-17T15:38:43,400 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for fdca08c93967309fb9c5dcd33e10e7d5 in 1282ms, sequenceid=224, compaction requested=true 2024-11-17T15:38:43,400 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fdca08c93967309fb9c5dcd33e10e7d5: 2024-11-17T15:38:43,400 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fdca08c93967309fb9c5dcd33e10e7d5:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T15:38:43,400 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T15:38:43,400 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fdca08c93967309fb9c5dcd33e10e7d5:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T15:38:43,400 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T15:38:43,400 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fdca08c93967309fb9c5dcd33e10e7d5:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T15:38:43,400 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T15:38:43,400 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T15:38:43,400 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T15:38:43,401 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T15:38:43,401 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111150 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T15:38:43,401 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1541): fdca08c93967309fb9c5dcd33e10e7d5/A is initiating minor compaction (all files) 2024-11-17T15:38:43,401 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1541): fdca08c93967309fb9c5dcd33e10e7d5/B is initiating minor compaction (all files) 2024-11-17T15:38:43,401 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fdca08c93967309fb9c5dcd33e10e7d5/A in TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. 2024-11-17T15:38:43,401 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fdca08c93967309fb9c5dcd33e10e7d5/B in TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. 2024-11-17T15:38:43,401 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/A/f4cd3f0ce3f74b5483bf2d7e2ca41145, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/A/f36338c0bab649a5a44aba1189f1c29f, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/A/c015e8481c0f480097986019990d3c16] into tmpdir=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp, totalSize=108.5 K 2024-11-17T15:38:43,401 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/B/7214dc60e22340c8a33ba09a2c775ba0, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/B/515bd0daef4f4bd3a3dece8c33f0f44b, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/B/beef8fa34b8b4649a1b7bfe875714457] into tmpdir=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp, totalSize=36.0 K 2024-11-17T15:38:43,402 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. 2024-11-17T15:38:43,402 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. files: [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/A/f4cd3f0ce3f74b5483bf2d7e2ca41145, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/A/f36338c0bab649a5a44aba1189f1c29f, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/A/c015e8481c0f480097986019990d3c16] 2024-11-17T15:38:43,402 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting 7214dc60e22340c8a33ba09a2c775ba0, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1731857916908 2024-11-17T15:38:43,402 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting f4cd3f0ce3f74b5483bf2d7e2ca41145, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1731857916908 2024-11-17T15:38:43,402 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting 515bd0daef4f4bd3a3dece8c33f0f44b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1731857919721 2024-11-17T15:38:43,402 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting f36338c0bab649a5a44aba1189f1c29f, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1731857919721 2024-11-17T15:38:43,402 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting beef8fa34b8b4649a1b7bfe875714457, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1731857920930 2024-11-17T15:38:43,402 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting c015e8481c0f480097986019990d3c16, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1731857920927 2024-11-17T15:38:43,407 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=fdca08c93967309fb9c5dcd33e10e7d5] 2024-11-17T15:38:43,409 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fdca08c93967309fb9c5dcd33e10e7d5#B#compaction#347 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T15:38:43,409 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/B/b4244e3f62074a4f837d53b0f31cb0a8 is 50, key is test_row_0/B:col10/1731857922118/Put/seqid=0 2024-11-17T15:38:43,409 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411173b7c3bb1a471463aa3cf739c47bbd972_fdca08c93967309fb9c5dcd33e10e7d5 store=[table=TestAcidGuarantees family=A region=fdca08c93967309fb9c5dcd33e10e7d5] 2024-11-17T15:38:43,411 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411173b7c3bb1a471463aa3cf739c47bbd972_fdca08c93967309fb9c5dcd33e10e7d5, store=[table=TestAcidGuarantees family=A region=fdca08c93967309fb9c5dcd33e10e7d5] 2024-11-17T15:38:43,412 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411173b7c3bb1a471463aa3cf739c47bbd972_fdca08c93967309fb9c5dcd33e10e7d5 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=fdca08c93967309fb9c5dcd33e10e7d5] 2024-11-17T15:38:43,413 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742241_1417 (size=12254) 2024-11-17T15:38:43,416 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-17T15:38:43,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. 2024-11-17T15:38:43,416 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2902): Flushing fdca08c93967309fb9c5dcd33e10e7d5 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-17T15:38:43,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742242_1418 (size=4469) 2024-11-17T15:38:43,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117d42f43160a9347ef8959d2e12d7b9b17_fdca08c93967309fb9c5dcd33e10e7d5 is 50, key is test_row_0/A:col10/1731857922181/Put/seqid=0 2024-11-17T15:38:43,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742243_1419 (size=12304) 2024-11-17T15:38:43,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-17T15:38:43,817 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/B/b4244e3f62074a4f837d53b0f31cb0a8 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/B/b4244e3f62074a4f837d53b0f31cb0a8 2024-11-17T15:38:43,822 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fdca08c93967309fb9c5dcd33e10e7d5#A#compaction#348 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T15:38:43,822 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/A/eed1a8d1d230408c8c4208ffc5e0c86b is 175, key is test_row_0/A:col10/1731857922118/Put/seqid=0 2024-11-17T15:38:43,823 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fdca08c93967309fb9c5dcd33e10e7d5/B of fdca08c93967309fb9c5dcd33e10e7d5 into b4244e3f62074a4f837d53b0f31cb0a8(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T15:38:43,823 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fdca08c93967309fb9c5dcd33e10e7d5: 2024-11-17T15:38:43,823 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., storeName=fdca08c93967309fb9c5dcd33e10e7d5/B, priority=13, startTime=1731857923400; duration=0sec 2024-11-17T15:38:43,823 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T15:38:43,823 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fdca08c93967309fb9c5dcd33e10e7d5:B 2024-11-17T15:38:43,823 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T15:38:43,824 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T15:38:43,825 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1541): fdca08c93967309fb9c5dcd33e10e7d5/C is initiating minor compaction (all files) 2024-11-17T15:38:43,825 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fdca08c93967309fb9c5dcd33e10e7d5/C in TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. 2024-11-17T15:38:43,825 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/C/12b2b5fe96704220a0fd9d763db8313d, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/C/0302936a8a8f43098144e453aec70a0c, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/C/86e57b9113084e9da7c4b9fba8572cd1] into tmpdir=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp, totalSize=36.0 K 2024-11-17T15:38:43,825 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting 12b2b5fe96704220a0fd9d763db8313d, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1731857916908 2024-11-17T15:38:43,825 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting 0302936a8a8f43098144e453aec70a0c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1731857919721 2024-11-17T15:38:43,828 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting 86e57b9113084e9da7c4b9fba8572cd1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1731857920930 2024-11-17T15:38:43,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:43,836 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117d42f43160a9347ef8959d2e12d7b9b17_fdca08c93967309fb9c5dcd33e10e7d5 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117d42f43160a9347ef8959d2e12d7b9b17_fdca08c93967309fb9c5dcd33e10e7d5 2024-11-17T15:38:43,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/A/e8b32c94510b440f842844d2cd5a05a2, store: [table=TestAcidGuarantees family=A region=fdca08c93967309fb9c5dcd33e10e7d5] 2024-11-17T15:38:43,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/A/e8b32c94510b440f842844d2cd5a05a2 is 175, key is test_row_0/A:col10/1731857922181/Put/seqid=0 2024-11-17T15:38:43,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742244_1420 (size=31208) 2024-11-17T15:38:43,843 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fdca08c93967309fb9c5dcd33e10e7d5#C#compaction#350 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T15:38:43,843 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/C/287fc6d564864557b467f236add9de83 is 50, key is test_row_0/C:col10/1731857922118/Put/seqid=0 2024-11-17T15:38:43,847 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/A/eed1a8d1d230408c8c4208ffc5e0c86b as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/A/eed1a8d1d230408c8c4208ffc5e0c86b 2024-11-17T15:38:43,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742246_1422 (size=12254) 2024-11-17T15:38:43,852 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fdca08c93967309fb9c5dcd33e10e7d5/A of fdca08c93967309fb9c5dcd33e10e7d5 into eed1a8d1d230408c8c4208ffc5e0c86b(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T15:38:43,852 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fdca08c93967309fb9c5dcd33e10e7d5: 2024-11-17T15:38:43,852 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., storeName=fdca08c93967309fb9c5dcd33e10e7d5/A, priority=13, startTime=1731857923400; duration=0sec 2024-11-17T15:38:43,854 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T15:38:43,854 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fdca08c93967309fb9c5dcd33e10e7d5:A 2024-11-17T15:38:43,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742245_1421 (size=31105) 2024-11-17T15:38:43,857 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/C/287fc6d564864557b467f236add9de83 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/C/287fc6d564864557b467f236add9de83 2024-11-17T15:38:43,863 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fdca08c93967309fb9c5dcd33e10e7d5/C of fdca08c93967309fb9c5dcd33e10e7d5 into 287fc6d564864557b467f236add9de83(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T15:38:43,863 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fdca08c93967309fb9c5dcd33e10e7d5: 2024-11-17T15:38:43,863 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., storeName=fdca08c93967309fb9c5dcd33e10e7d5/C, priority=13, startTime=1731857923400; duration=0sec 2024-11-17T15:38:43,863 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T15:38:43,863 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fdca08c93967309fb9c5dcd33e10e7d5:C 2024-11-17T15:38:43,976 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. as already flushing 2024-11-17T15:38:43,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(8855): Flush requested on fdca08c93967309fb9c5dcd33e10e7d5 2024-11-17T15:38:44,029 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:44,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49654 deadline: 1731857984016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:44,030 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:44,030 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:44,030 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:44,030 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:44,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49690 deadline: 1731857984021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:44,030 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:44,030 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:44,030 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:44,141 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:44,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49654 deadline: 1731857984133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:44,141 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:44,141 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:44,141 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:44,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49690 deadline: 1731857984133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:44,141 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:44,141 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:44,142 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:44,142 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:44,256 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=248, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/A/e8b32c94510b440f842844d2cd5a05a2 2024-11-17T15:38:44,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/B/4a408214c3ea49dd9696d1b0b36ba6eb is 50, key is test_row_0/B:col10/1731857922181/Put/seqid=0 2024-11-17T15:38:44,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742247_1423 (size=12151) 2024-11-17T15:38:44,362 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:44,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49654 deadline: 1731857984353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:44,362 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:44,362 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:44,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49690 deadline: 1731857984354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:44,362 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:44,362 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:44,363 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:44,363 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:44,363 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:44,411 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:44,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49700 deadline: 1731857984404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:44,412 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:44,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49680 deadline: 1731857984404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:44,412 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 7a780d55532c,32955,1731857800648 for 'test_row_0' in fdca08c93967309fb9c5dcd33e10e7d5 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2202 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T15:38:44,413 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:44,413 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:44,413 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:44,413 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 7a780d55532c,32955,1731857800648 for 'test_row_1' in fdca08c93967309fb9c5dcd33e10e7d5 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2202 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T15:38:44,414 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:44,414 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:44,414 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:44,421 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:44,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49652 deadline: 1731857984413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:44,421 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 7a780d55532c,32955,1731857800648 for 'test_row_1' in fdca08c93967309fb9c5dcd33e10e7d5 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2219 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T15:38:44,422 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:44,422 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:44,422 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:44,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-17T15:38:44,679 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/B/4a408214c3ea49dd9696d1b0b36ba6eb 2024-11-17T15:38:44,682 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:44,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49654 deadline: 1731857984673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:44,683 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:44,683 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:44,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49690 deadline: 1731857984673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:44,683 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:44,683 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:44,683 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:44,683 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:44,683 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:44,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/C/b9e128cc5e704ca3b07d4046e82d01a2 is 50, key is test_row_0/C:col10/1731857922181/Put/seqid=0 2024-11-17T15:38:44,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742248_1424 (size=12151) 2024-11-17T15:38:44,698 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/C/b9e128cc5e704ca3b07d4046e82d01a2 2024-11-17T15:38:44,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/A/e8b32c94510b440f842844d2cd5a05a2 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/A/e8b32c94510b440f842844d2cd5a05a2 2024-11-17T15:38:44,706 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/A/e8b32c94510b440f842844d2cd5a05a2, entries=150, sequenceid=248, filesize=30.4 K 2024-11-17T15:38:44,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/B/4a408214c3ea49dd9696d1b0b36ba6eb as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/B/4a408214c3ea49dd9696d1b0b36ba6eb 2024-11-17T15:38:44,712 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/B/4a408214c3ea49dd9696d1b0b36ba6eb, entries=150, sequenceid=248, filesize=11.9 K 2024-11-17T15:38:44,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/C/b9e128cc5e704ca3b07d4046e82d01a2 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/C/b9e128cc5e704ca3b07d4046e82d01a2 2024-11-17T15:38:44,717 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/C/b9e128cc5e704ca3b07d4046e82d01a2, entries=150, sequenceid=248, filesize=11.9 K 2024-11-17T15:38:44,718 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for fdca08c93967309fb9c5dcd33e10e7d5 in 1302ms, sequenceid=248, compaction requested=false 2024-11-17T15:38:44,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2603): Flush status journal for fdca08c93967309fb9c5dcd33e10e7d5: 2024-11-17T15:38:44,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. 2024-11-17T15:38:44,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=114 2024-11-17T15:38:44,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.HMaster(4169): Remote procedure done, pid=114 2024-11-17T15:38:44,722 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-11-17T15:38:44,722 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3830 sec 2024-11-17T15:38:44,725 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees in 2.3890 sec 2024-11-17T15:38:45,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(8855): Flush requested on fdca08c93967309fb9c5dcd33e10e7d5 2024-11-17T15:38:45,197 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T15:38:45,198 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fdca08c93967309fb9c5dcd33e10e7d5 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-17T15:38:45,204 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411178d420cfce30b4d9abdc802491d253b8d_fdca08c93967309fb9c5dcd33e10e7d5 is 50, key is test_row_0/A:col10/1731857925196/Put/seqid=0 2024-11-17T15:38:45,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742249_1425 (size=14994) 2024-11-17T15:38:45,208 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:45,212 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411178d420cfce30b4d9abdc802491d253b8d_fdca08c93967309fb9c5dcd33e10e7d5 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411178d420cfce30b4d9abdc802491d253b8d_fdca08c93967309fb9c5dcd33e10e7d5 2024-11-17T15:38:45,213 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/A/9089b31ca4564108829ca1e6acc55a12, store: [table=TestAcidGuarantees family=A region=fdca08c93967309fb9c5dcd33e10e7d5] 2024-11-17T15:38:45,213 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/A/9089b31ca4564108829ca1e6acc55a12 is 175, key is test_row_0/A:col10/1731857925196/Put/seqid=0 2024-11-17T15:38:45,216 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742250_1426 (size=39949) 2024-11-17T15:38:45,299 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:45,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49690 deadline: 1731857985288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:45,299 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:45,299 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:45,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49654 deadline: 1731857985293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:45,299 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:45,299 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:45,299 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:45,299 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:45,300 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:45,408 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:45,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49690 deadline: 1731857985402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:45,408 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:45,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49654 deadline: 1731857985403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:45,408 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:45,408 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:45,408 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:45,408 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:45,409 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:45,409 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:45,545 INFO [Thread-1615 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1615. 2024-11-17T15:38:45,545 DEBUG [Thread-1615 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T15:38:45,545 DEBUG [Thread-1615 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:38:45,545 DEBUG [Thread-1615 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:38:45,545 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T15:38:45,545 INFO [Thread-1617 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1617. 2024-11-17T15:38:45,545 DEBUG [Thread-1617 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T15:38:45,545 DEBUG [Thread-1617 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:38:45,545 DEBUG [Thread-1617 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:38:45,545 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T15:38:45,552 INFO [Thread-1619 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1619. 2024-11-17T15:38:45,552 DEBUG [Thread-1619 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T15:38:45,552 DEBUG [Thread-1619 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:38:45,552 DEBUG [Thread-1619 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:38:45,552 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T15:38:45,555 INFO [Thread-1611 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1611. 2024-11-17T15:38:45,555 DEBUG [Thread-1611 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T15:38:45,555 DEBUG [Thread-1611 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:38:45,555 DEBUG [Thread-1611 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:38:45,555 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T15:38:45,555 INFO [Thread-1613 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1613. 2024-11-17T15:38:45,555 DEBUG [Thread-1613 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T15:38:45,555 DEBUG [Thread-1613 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:38:45,555 DEBUG [Thread-1613 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:38:45,555 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T15:38:45,614 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:45,614 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:45,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49690 deadline: 1731857985613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:45,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49654 deadline: 1731857985613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:45,614 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:45,614 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:45,614 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:45,614 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:45,614 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:45,614 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:45,617 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=264, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/A/9089b31ca4564108829ca1e6acc55a12 2024-11-17T15:38:45,624 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/B/2d137f6bfd9b415d83ddfc866afb84a6 is 50, key is test_row_0/B:col10/1731857925196/Put/seqid=0 2024-11-17T15:38:45,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742251_1427 (size=12301) 2024-11-17T15:38:45,923 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:45,923 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:45,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49690 deadline: 1731857985923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:45,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49654 deadline: 1731857985923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:45,924 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:45,924 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:45,924 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:45,924 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:45,924 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:45,924 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:46,027 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=264 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/B/2d137f6bfd9b415d83ddfc866afb84a6 2024-11-17T15:38:46,033 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/C/431e7549b4a44a9491cbff4199a0211b is 50, key is test_row_0/C:col10/1731857925196/Put/seqid=0 2024-11-17T15:38:46,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742252_1428 (size=12301) 2024-11-17T15:38:46,423 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:46,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49700 deadline: 1731857986423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:46,424 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 7a780d55532c,32955,1731857800648 for 'test_row_0' in fdca08c93967309fb9c5dcd33e10e7d5 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4214 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T15:38:46,424 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:46,424 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:46,425 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:46,434 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:46,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49680 deadline: 1731857986433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:46,434 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:46,434 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:46,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49690 deadline: 1731857986433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:46,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49654 deadline: 1731857986434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:46,434 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:46,434 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:46,434 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:46,434 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:46,434 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:46,434 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:46,434 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 7a780d55532c,32955,1731857800648 for 'test_row_1' in fdca08c93967309fb9c5dcd33e10e7d5 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4222 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T15:38:46,434 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:46,434 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:46,434 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:46,437 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=264 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/C/431e7549b4a44a9491cbff4199a0211b 2024-11-17T15:38:46,440 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/A/9089b31ca4564108829ca1e6acc55a12 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/A/9089b31ca4564108829ca1e6acc55a12 2024-11-17T15:38:46,443 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:46,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49652 deadline: 1731857986443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:46,443 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/A/9089b31ca4564108829ca1e6acc55a12, entries=200, sequenceid=264, filesize=39.0 K 2024-11-17T15:38:46,443 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 7a780d55532c,32955,1731857800648 for 'test_row_1' in fdca08c93967309fb9c5dcd33e10e7d5 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4241 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T15:38:46,444 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:46,444 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/B/2d137f6bfd9b415d83ddfc866afb84a6 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/B/2d137f6bfd9b415d83ddfc866afb84a6 2024-11-17T15:38:46,444 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:46,444 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:38:46,447 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/B/2d137f6bfd9b415d83ddfc866afb84a6, entries=150, sequenceid=264, filesize=12.0 K 2024-11-17T15:38:46,447 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/C/431e7549b4a44a9491cbff4199a0211b as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/C/431e7549b4a44a9491cbff4199a0211b 2024-11-17T15:38:46,450 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/C/431e7549b4a44a9491cbff4199a0211b, entries=150, sequenceid=264, filesize=12.0 K 2024-11-17T15:38:46,451 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for fdca08c93967309fb9c5dcd33e10e7d5 in 1254ms, sequenceid=264, compaction requested=true 2024-11-17T15:38:46,451 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fdca08c93967309fb9c5dcd33e10e7d5: 2024-11-17T15:38:46,451 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fdca08c93967309fb9c5dcd33e10e7d5:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T15:38:46,451 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T15:38:46,451 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fdca08c93967309fb9c5dcd33e10e7d5:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T15:38:46,451 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T15:38:46,451 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fdca08c93967309fb9c5dcd33e10e7d5:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T15:38:46,451 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T15:38:46,451 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T15:38:46,451 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T15:38:46,452 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102262 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T15:38:46,452 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36706 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T15:38:46,452 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1541): fdca08c93967309fb9c5dcd33e10e7d5/A is initiating minor compaction (all files) 2024-11-17T15:38:46,452 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1541): fdca08c93967309fb9c5dcd33e10e7d5/B is initiating minor compaction (all files) 2024-11-17T15:38:46,452 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fdca08c93967309fb9c5dcd33e10e7d5/A in TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. 2024-11-17T15:38:46,452 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fdca08c93967309fb9c5dcd33e10e7d5/B in TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. 2024-11-17T15:38:46,452 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/A/eed1a8d1d230408c8c4208ffc5e0c86b, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/A/e8b32c94510b440f842844d2cd5a05a2, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/A/9089b31ca4564108829ca1e6acc55a12] into tmpdir=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp, totalSize=99.9 K 2024-11-17T15:38:46,452 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/B/b4244e3f62074a4f837d53b0f31cb0a8, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/B/4a408214c3ea49dd9696d1b0b36ba6eb, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/B/2d137f6bfd9b415d83ddfc866afb84a6] into tmpdir=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp, totalSize=35.8 K 2024-11-17T15:38:46,452 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. 2024-11-17T15:38:46,452 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. files: [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/A/eed1a8d1d230408c8c4208ffc5e0c86b, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/A/e8b32c94510b440f842844d2cd5a05a2, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/A/9089b31ca4564108829ca1e6acc55a12] 2024-11-17T15:38:46,452 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting b4244e3f62074a4f837d53b0f31cb0a8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1731857920930 2024-11-17T15:38:46,452 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting eed1a8d1d230408c8c4208ffc5e0c86b, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1731857920930 2024-11-17T15:38:46,453 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting 4a408214c3ea49dd9696d1b0b36ba6eb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1731857922181 2024-11-17T15:38:46,453 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting e8b32c94510b440f842844d2cd5a05a2, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1731857922181 2024-11-17T15:38:46,453 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting 2d137f6bfd9b415d83ddfc866afb84a6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1731857924006 2024-11-17T15:38:46,453 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9089b31ca4564108829ca1e6acc55a12, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1731857924006 2024-11-17T15:38:46,458 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=fdca08c93967309fb9c5dcd33e10e7d5] 2024-11-17T15:38:46,458 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fdca08c93967309fb9c5dcd33e10e7d5#B#compaction#356 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T15:38:46,459 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/B/5f370a262d0f4994b58140d639b52ed2 is 50, key is test_row_0/B:col10/1731857925196/Put/seqid=0 2024-11-17T15:38:46,459 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411177d5f7efabd99446a85a09090abcc28f2_fdca08c93967309fb9c5dcd33e10e7d5 store=[table=TestAcidGuarantees family=A region=fdca08c93967309fb9c5dcd33e10e7d5] 2024-11-17T15:38:46,461 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411177d5f7efabd99446a85a09090abcc28f2_fdca08c93967309fb9c5dcd33e10e7d5, store=[table=TestAcidGuarantees family=A region=fdca08c93967309fb9c5dcd33e10e7d5] 2024-11-17T15:38:46,461 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411177d5f7efabd99446a85a09090abcc28f2_fdca08c93967309fb9c5dcd33e10e7d5 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=fdca08c93967309fb9c5dcd33e10e7d5] 2024-11-17T15:38:46,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742253_1429 (size=12507) 2024-11-17T15:38:46,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742254_1430 (size=4469) 2024-11-17T15:38:46,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-17T15:38:46,504 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T15:38:46,865 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fdca08c93967309fb9c5dcd33e10e7d5#A#compaction#357 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T15:38:46,866 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/A/ea17370150194f7cbb758937addf2db4 is 175, key is test_row_0/A:col10/1731857925196/Put/seqid=0 2024-11-17T15:38:46,867 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/B/5f370a262d0f4994b58140d639b52ed2 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/B/5f370a262d0f4994b58140d639b52ed2 2024-11-17T15:38:46,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742255_1431 (size=31461) 2024-11-17T15:38:46,871 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fdca08c93967309fb9c5dcd33e10e7d5/B of fdca08c93967309fb9c5dcd33e10e7d5 into 5f370a262d0f4994b58140d639b52ed2(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T15:38:46,871 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fdca08c93967309fb9c5dcd33e10e7d5: 2024-11-17T15:38:46,871 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., storeName=fdca08c93967309fb9c5dcd33e10e7d5/B, priority=13, startTime=1731857926451; duration=0sec 2024-11-17T15:38:46,871 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T15:38:46,871 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fdca08c93967309fb9c5dcd33e10e7d5:B 2024-11-17T15:38:46,872 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T15:38:46,872 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36706 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T15:38:46,872 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1541): fdca08c93967309fb9c5dcd33e10e7d5/C is initiating minor compaction (all files) 2024-11-17T15:38:46,872 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fdca08c93967309fb9c5dcd33e10e7d5/C in TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. 2024-11-17T15:38:46,872 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/C/287fc6d564864557b467f236add9de83, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/C/b9e128cc5e704ca3b07d4046e82d01a2, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/C/431e7549b4a44a9491cbff4199a0211b] into tmpdir=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp, totalSize=35.8 K 2024-11-17T15:38:46,873 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting 287fc6d564864557b467f236add9de83, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1731857920930 2024-11-17T15:38:46,873 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting b9e128cc5e704ca3b07d4046e82d01a2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1731857922181 2024-11-17T15:38:46,873 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting 431e7549b4a44a9491cbff4199a0211b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1731857924006 2024-11-17T15:38:46,878 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fdca08c93967309fb9c5dcd33e10e7d5#C#compaction#358 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T15:38:46,878 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/C/a5ddf924f964453d8e73d25e7ae1b918 is 50, key is test_row_0/C:col10/1731857925196/Put/seqid=0 2024-11-17T15:38:46,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742256_1432 (size=12507) 2024-11-17T15:38:47,273 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/A/ea17370150194f7cbb758937addf2db4 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/A/ea17370150194f7cbb758937addf2db4 2024-11-17T15:38:47,277 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fdca08c93967309fb9c5dcd33e10e7d5/A of fdca08c93967309fb9c5dcd33e10e7d5 into ea17370150194f7cbb758937addf2db4(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T15:38:47,277 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fdca08c93967309fb9c5dcd33e10e7d5: 2024-11-17T15:38:47,277 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., storeName=fdca08c93967309fb9c5dcd33e10e7d5/A, priority=13, startTime=1731857926451; duration=0sec 2024-11-17T15:38:47,277 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T15:38:47,277 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fdca08c93967309fb9c5dcd33e10e7d5:A 2024-11-17T15:38:47,285 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/C/a5ddf924f964453d8e73d25e7ae1b918 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/C/a5ddf924f964453d8e73d25e7ae1b918 2024-11-17T15:38:47,289 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fdca08c93967309fb9c5dcd33e10e7d5/C of fdca08c93967309fb9c5dcd33e10e7d5 into a5ddf924f964453d8e73d25e7ae1b918(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T15:38:47,289 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fdca08c93967309fb9c5dcd33e10e7d5: 2024-11-17T15:38:47,289 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., storeName=fdca08c93967309fb9c5dcd33e10e7d5/C, priority=13, startTime=1731857926451; duration=0sec 2024-11-17T15:38:47,289 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T15:38:47,289 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fdca08c93967309fb9c5dcd33e10e7d5:C 2024-11-17T15:38:47,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(8855): Flush requested on fdca08c93967309fb9c5dcd33e10e7d5 2024-11-17T15:38:47,445 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T15:38:47,445 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fdca08c93967309fb9c5dcd33e10e7d5 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-17T15:38:47,445 INFO [Thread-1602 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1602. 2024-11-17T15:38:47,445 INFO [Thread-1608 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1608. 2024-11-17T15:38:47,445 DEBUG [Thread-1602 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T15:38:47,445 DEBUG [Thread-1608 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T15:38:47,445 DEBUG [Thread-1602 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:38:47,445 DEBUG [Thread-1608 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:38:47,445 DEBUG [Thread-1608 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:38:47,445 DEBUG [Thread-1602 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:38:47,445 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T15:38:47,445 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T15:38:47,450 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117c879c52bfdac470e8ffce11604a7989a_fdca08c93967309fb9c5dcd33e10e7d5 is 50, key is test_row_0/A:col10/1731857925291/Put/seqid=0 2024-11-17T15:38:47,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742257_1433 (size=12454) 2024-11-17T15:38:47,854 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:47,857 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117c879c52bfdac470e8ffce11604a7989a_fdca08c93967309fb9c5dcd33e10e7d5 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117c879c52bfdac470e8ffce11604a7989a_fdca08c93967309fb9c5dcd33e10e7d5 2024-11-17T15:38:47,858 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/A/e323835ddf5a433fbcdf48a4744bcc36, store: [table=TestAcidGuarantees family=A region=fdca08c93967309fb9c5dcd33e10e7d5] 2024-11-17T15:38:47,859 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/A/e323835ddf5a433fbcdf48a4744bcc36 is 175, key is test_row_0/A:col10/1731857925291/Put/seqid=0 2024-11-17T15:38:47,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742258_1434 (size=31255) 2024-11-17T15:38:48,262 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=292, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/A/e323835ddf5a433fbcdf48a4744bcc36 2024-11-17T15:38:48,268 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/B/ec53ba73b00d4a1fb0c651768a09c1e5 is 50, key is test_row_0/B:col10/1731857925291/Put/seqid=0 2024-11-17T15:38:48,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742259_1435 (size=12301) 2024-11-17T15:38:48,671 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/B/ec53ba73b00d4a1fb0c651768a09c1e5 2024-11-17T15:38:48,677 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/C/ca75cd8bedf647c99e87ee04a1c2e409 is 50, key is test_row_0/C:col10/1731857925291/Put/seqid=0 2024-11-17T15:38:48,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742260_1436 (size=12301) 2024-11-17T15:38:49,080 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/C/ca75cd8bedf647c99e87ee04a1c2e409 2024-11-17T15:38:49,084 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/A/e323835ddf5a433fbcdf48a4744bcc36 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/A/e323835ddf5a433fbcdf48a4744bcc36 2024-11-17T15:38:49,087 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/A/e323835ddf5a433fbcdf48a4744bcc36, entries=150, sequenceid=292, filesize=30.5 K 2024-11-17T15:38:49,087 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/B/ec53ba73b00d4a1fb0c651768a09c1e5 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/B/ec53ba73b00d4a1fb0c651768a09c1e5 2024-11-17T15:38:49,090 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/B/ec53ba73b00d4a1fb0c651768a09c1e5, entries=150, sequenceid=292, filesize=12.0 K 2024-11-17T15:38:49,090 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/C/ca75cd8bedf647c99e87ee04a1c2e409 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/C/ca75cd8bedf647c99e87ee04a1c2e409 2024-11-17T15:38:49,093 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/C/ca75cd8bedf647c99e87ee04a1c2e409, entries=150, sequenceid=292, filesize=12.0 K 2024-11-17T15:38:49,094 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=0 B/0 for fdca08c93967309fb9c5dcd33e10e7d5 in 1649ms, sequenceid=292, compaction requested=false 2024-11-17T15:38:49,094 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fdca08c93967309fb9c5dcd33e10e7d5: 2024-11-17T15:38:50,444 INFO [Thread-1606 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1606. 2024-11-17T15:38:50,445 DEBUG [Thread-1606 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T15:38:50,445 DEBUG [Thread-1606 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:38:50,445 DEBUG [Thread-1606 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:38:50,445 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T15:38:50,465 INFO [Thread-1600 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1600. 2024-11-17T15:38:50,465 INFO [Thread-1604 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1604. 2024-11-17T15:38:50,465 DEBUG [Thread-1604 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T15:38:50,465 DEBUG [Thread-1600 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T15:38:50,465 DEBUG [Thread-1604 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:38:50,465 DEBUG [Thread-1600 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:38:50,465 DEBUG [Thread-1604 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:38:50,465 DEBUG [Thread-1600 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:38:50,465 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T15:38:50,465 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T15:38:50,465 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-17T15:38:50,465 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 43 2024-11-17T15:38:50,465 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 55 2024-11-17T15:38:50,465 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 46 2024-11-17T15:38:50,465 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 41 2024-11-17T15:38:50,465 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 43 2024-11-17T15:38:50,465 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-17T15:38:50,465 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-17T15:38:50,465 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1129 2024-11-17T15:38:50,465 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3384 rows 2024-11-17T15:38:50,465 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1128 2024-11-17T15:38:50,465 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3378 rows 2024-11-17T15:38:50,465 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1123 2024-11-17T15:38:50,465 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3368 rows 2024-11-17T15:38:50,465 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1128 2024-11-17T15:38:50,465 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3378 rows 2024-11-17T15:38:50,465 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1130 2024-11-17T15:38:50,465 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3378 rows 2024-11-17T15:38:50,465 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-17T15:38:50,465 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-17T15:38:50,465 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:38:50,465 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:38:50,465 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T15:38:50,466 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-17T15:38:50,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] procedure2.ProcedureExecutor(1139): Stored pid=115, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-17T15:38:50,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-17T15:38:50,469 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731857930469"}]},"ts":"1731857930469"} 2024-11-17T15:38:50,470 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-17T15:38:50,470 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-17T15:38:50,470 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-17T15:38:50,472 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=117, ppid=116, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fdca08c93967309fb9c5dcd33e10e7d5, UNASSIGN}] 2024-11-17T15:38:50,472 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=117, ppid=116, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fdca08c93967309fb9c5dcd33e10e7d5, UNASSIGN 2024-11-17T15:38:50,473 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=117 updating hbase:meta row=fdca08c93967309fb9c5dcd33e10e7d5, regionState=CLOSING, regionLocation=7a780d55532c,32955,1731857800648 2024-11-17T15:38:50,474 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=117, ppid=116, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=fdca08c93967309fb9c5dcd33e10e7d5, UNASSIGN because future has completed 2024-11-17T15:38:50,474 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-17T15:38:50,474 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE, hasLock=false; CloseRegionProcedure fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648}] 2024-11-17T15:38:50,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-17T15:38:50,626 INFO [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] handler.UnassignRegionHandler(122): Close fdca08c93967309fb9c5dcd33e10e7d5 2024-11-17T15:38:50,626 DEBUG [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-17T15:38:50,626 DEBUG [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(1722): Closing fdca08c93967309fb9c5dcd33e10e7d5, disabling compactions & flushes 2024-11-17T15:38:50,626 INFO [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. 2024-11-17T15:38:50,626 DEBUG [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. 2024-11-17T15:38:50,626 DEBUG [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. after waiting 0 ms 2024-11-17T15:38:50,626 DEBUG [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. 2024-11-17T15:38:50,626 INFO [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(2902): Flushing fdca08c93967309fb9c5dcd33e10e7d5 3/3 column families, dataSize=20.13 KB heapSize=53.48 KB 2024-11-17T15:38:50,631 DEBUG [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117f65c35ffecde42159224b61fb1c2c720_fdca08c93967309fb9c5dcd33e10e7d5 is 50, key is test_row_0/A:col10/1731857930443/Put/seqid=0 2024-11-17T15:38:50,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742261_1437 (size=9914) 2024-11-17T15:38:50,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-17T15:38:51,035 DEBUG [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:51,038 INFO [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117f65c35ffecde42159224b61fb1c2c720_fdca08c93967309fb9c5dcd33e10e7d5 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117f65c35ffecde42159224b61fb1c2c720_fdca08c93967309fb9c5dcd33e10e7d5 2024-11-17T15:38:51,039 DEBUG [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/A/3665514f4b79422cb7964eda189f49a2, store: [table=TestAcidGuarantees family=A region=fdca08c93967309fb9c5dcd33e10e7d5] 2024-11-17T15:38:51,039 DEBUG [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/A/3665514f4b79422cb7964eda189f49a2 is 175, key is test_row_0/A:col10/1731857930443/Put/seqid=0 2024-11-17T15:38:51,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742262_1438 (size=22561) 2024-11-17T15:38:51,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-17T15:38:51,443 INFO [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=298, memsize=6.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/A/3665514f4b79422cb7964eda189f49a2 2024-11-17T15:38:51,449 DEBUG [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/B/f4d73f2d3698425d9d2fa2a5fd13711d is 50, key is test_row_0/B:col10/1731857930443/Put/seqid=0 2024-11-17T15:38:51,451 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742263_1439 (size=9857) 2024-11-17T15:38:51,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-17T15:38:51,852 INFO [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=298 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/B/f4d73f2d3698425d9d2fa2a5fd13711d 2024-11-17T15:38:51,858 DEBUG [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/C/7037bda766474747aada5fc95e34e90b is 50, key is test_row_0/C:col10/1731857930443/Put/seqid=0 2024-11-17T15:38:51,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742264_1440 (size=9857) 2024-11-17T15:38:52,261 INFO [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=298 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/C/7037bda766474747aada5fc95e34e90b 2024-11-17T15:38:52,265 DEBUG [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/A/3665514f4b79422cb7964eda189f49a2 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/A/3665514f4b79422cb7964eda189f49a2 2024-11-17T15:38:52,269 INFO [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/A/3665514f4b79422cb7964eda189f49a2, entries=100, sequenceid=298, filesize=22.0 K 2024-11-17T15:38:52,269 DEBUG [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/B/f4d73f2d3698425d9d2fa2a5fd13711d as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/B/f4d73f2d3698425d9d2fa2a5fd13711d 2024-11-17T15:38:52,273 INFO [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/B/f4d73f2d3698425d9d2fa2a5fd13711d, entries=100, sequenceid=298, filesize=9.6 K 2024-11-17T15:38:52,274 DEBUG [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/.tmp/C/7037bda766474747aada5fc95e34e90b as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/C/7037bda766474747aada5fc95e34e90b 2024-11-17T15:38:52,276 INFO [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/C/7037bda766474747aada5fc95e34e90b, entries=100, sequenceid=298, filesize=9.6 K 2024-11-17T15:38:52,277 INFO [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=0 B/0 for fdca08c93967309fb9c5dcd33e10e7d5 in 1651ms, sequenceid=298, compaction requested=true 2024-11-17T15:38:52,278 DEBUG [StoreCloser-TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/A/f4cd3f0ce3f74b5483bf2d7e2ca41145, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/A/f36338c0bab649a5a44aba1189f1c29f, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/A/c015e8481c0f480097986019990d3c16, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/A/eed1a8d1d230408c8c4208ffc5e0c86b, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/A/e8b32c94510b440f842844d2cd5a05a2, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/A/9089b31ca4564108829ca1e6acc55a12] to archive 2024-11-17T15:38:52,278 DEBUG [StoreCloser-TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-17T15:38:52,279 DEBUG [StoreCloser-TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/A/f4cd3f0ce3f74b5483bf2d7e2ca41145 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/A/f4cd3f0ce3f74b5483bf2d7e2ca41145 2024-11-17T15:38:52,280 DEBUG [StoreCloser-TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/A/f36338c0bab649a5a44aba1189f1c29f to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/A/f36338c0bab649a5a44aba1189f1c29f 2024-11-17T15:38:52,281 DEBUG [StoreCloser-TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/A/c015e8481c0f480097986019990d3c16 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/A/c015e8481c0f480097986019990d3c16 2024-11-17T15:38:52,282 DEBUG [StoreCloser-TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/A/eed1a8d1d230408c8c4208ffc5e0c86b to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/A/eed1a8d1d230408c8c4208ffc5e0c86b 2024-11-17T15:38:52,283 DEBUG [StoreCloser-TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/A/e8b32c94510b440f842844d2cd5a05a2 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/A/e8b32c94510b440f842844d2cd5a05a2 2024-11-17T15:38:52,283 DEBUG [StoreCloser-TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/A/9089b31ca4564108829ca1e6acc55a12 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/A/9089b31ca4564108829ca1e6acc55a12 2024-11-17T15:38:52,284 DEBUG [StoreCloser-TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/B/7214dc60e22340c8a33ba09a2c775ba0, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/B/515bd0daef4f4bd3a3dece8c33f0f44b, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/B/b4244e3f62074a4f837d53b0f31cb0a8, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/B/beef8fa34b8b4649a1b7bfe875714457, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/B/4a408214c3ea49dd9696d1b0b36ba6eb, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/B/2d137f6bfd9b415d83ddfc866afb84a6] to archive 2024-11-17T15:38:52,285 DEBUG [StoreCloser-TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-17T15:38:52,286 DEBUG [StoreCloser-TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/B/7214dc60e22340c8a33ba09a2c775ba0 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/B/7214dc60e22340c8a33ba09a2c775ba0 2024-11-17T15:38:52,287 DEBUG [StoreCloser-TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/B/515bd0daef4f4bd3a3dece8c33f0f44b to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/B/515bd0daef4f4bd3a3dece8c33f0f44b 2024-11-17T15:38:52,288 DEBUG [StoreCloser-TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/B/b4244e3f62074a4f837d53b0f31cb0a8 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/B/b4244e3f62074a4f837d53b0f31cb0a8 2024-11-17T15:38:52,288 DEBUG [StoreCloser-TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/B/beef8fa34b8b4649a1b7bfe875714457 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/B/beef8fa34b8b4649a1b7bfe875714457 2024-11-17T15:38:52,289 DEBUG [StoreCloser-TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/B/4a408214c3ea49dd9696d1b0b36ba6eb to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/B/4a408214c3ea49dd9696d1b0b36ba6eb 2024-11-17T15:38:52,290 DEBUG [StoreCloser-TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/B/2d137f6bfd9b415d83ddfc866afb84a6 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/B/2d137f6bfd9b415d83ddfc866afb84a6 2024-11-17T15:38:52,291 DEBUG [StoreCloser-TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/C/12b2b5fe96704220a0fd9d763db8313d, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/C/0302936a8a8f43098144e453aec70a0c, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/C/287fc6d564864557b467f236add9de83, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/C/86e57b9113084e9da7c4b9fba8572cd1, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/C/b9e128cc5e704ca3b07d4046e82d01a2, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/C/431e7549b4a44a9491cbff4199a0211b] to archive 2024-11-17T15:38:52,291 DEBUG [StoreCloser-TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-17T15:38:52,292 DEBUG [StoreCloser-TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/C/12b2b5fe96704220a0fd9d763db8313d to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/C/12b2b5fe96704220a0fd9d763db8313d 2024-11-17T15:38:52,293 DEBUG [StoreCloser-TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/C/0302936a8a8f43098144e453aec70a0c to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/C/0302936a8a8f43098144e453aec70a0c 2024-11-17T15:38:52,294 DEBUG [StoreCloser-TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/C/287fc6d564864557b467f236add9de83 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/C/287fc6d564864557b467f236add9de83 2024-11-17T15:38:52,295 DEBUG [StoreCloser-TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/C/86e57b9113084e9da7c4b9fba8572cd1 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/C/86e57b9113084e9da7c4b9fba8572cd1 2024-11-17T15:38:52,296 DEBUG [StoreCloser-TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/C/b9e128cc5e704ca3b07d4046e82d01a2 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/C/b9e128cc5e704ca3b07d4046e82d01a2 2024-11-17T15:38:52,296 DEBUG [StoreCloser-TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/C/431e7549b4a44a9491cbff4199a0211b to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/C/431e7549b4a44a9491cbff4199a0211b 2024-11-17T15:38:52,299 DEBUG [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/recovered.edits/301.seqid, newMaxSeqId=301, maxSeqId=4 2024-11-17T15:38:52,300 INFO [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. 2024-11-17T15:38:52,300 DEBUG [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(1676): Region close journal for fdca08c93967309fb9c5dcd33e10e7d5: Waiting for close lock at 1731857930626Running coprocessor pre-close hooks at 1731857930626Disabling compacts and flushes for region at 1731857930626Disabling writes for close at 1731857930626Obtaining lock to block concurrent updates at 1731857930626Preparing flush snapshotting stores in fdca08c93967309fb9c5dcd33e10e7d5 at 1731857930626Finished memstore snapshotting TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=54720, getOffHeapSize=0, getCellsCount=450 at 1731857930627 (+1 ms)Flushing stores of TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5. at 1731857930627Flushing fdca08c93967309fb9c5dcd33e10e7d5/A: creating writer at 1731857930627Flushing mob file fdca08c93967309fb9c5dcd33e10e7d5/A: appending metadata at 1731857930631 (+4 ms)Flushing mob file fdca08c93967309fb9c5dcd33e10e7d5/A: closing flushed file at 1731857930631Flushing fdca08c93967309fb9c5dcd33e10e7d5/A: appending metadata at 1731857931039 (+408 ms)Flushing fdca08c93967309fb9c5dcd33e10e7d5/A: closing flushed file at 1731857931039Flushing fdca08c93967309fb9c5dcd33e10e7d5/B: creating writer at 1731857931446 (+407 ms)Flushing fdca08c93967309fb9c5dcd33e10e7d5/B: appending metadata at 1731857931448 (+2 ms)Flushing fdca08c93967309fb9c5dcd33e10e7d5/B: closing flushed file at 1731857931448Flushing fdca08c93967309fb9c5dcd33e10e7d5/C: creating writer at 1731857931855 (+407 ms)Flushing fdca08c93967309fb9c5dcd33e10e7d5/C: appending metadata at 1731857931857 (+2 ms)Flushing fdca08c93967309fb9c5dcd33e10e7d5/C: closing flushed file at 1731857931857Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@30199673: reopening flushed file at 1731857932265 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@23c7d3d2: reopening flushed file at 1731857932269 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@63405f9a: reopening flushed file at 1731857932273 (+4 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=0 B/0 for fdca08c93967309fb9c5dcd33e10e7d5 in 1651ms, sequenceid=298, compaction requested=true at 1731857932277 (+4 ms)Writing region close event to WAL at 1731857932297 (+20 ms)Running coprocessor post-close hooks at 1731857932300 (+3 ms)Closed at 1731857932300 2024-11-17T15:38:52,302 INFO [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] handler.UnassignRegionHandler(157): Closed fdca08c93967309fb9c5dcd33e10e7d5 2024-11-17T15:38:52,302 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=117 updating hbase:meta row=fdca08c93967309fb9c5dcd33e10e7d5, regionState=CLOSED 2024-11-17T15:38:52,304 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=118, ppid=117, state=RUNNABLE, hasLock=false; CloseRegionProcedure fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 because future has completed 2024-11-17T15:38:52,306 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-11-17T15:38:52,306 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; CloseRegionProcedure fdca08c93967309fb9c5dcd33e10e7d5, server=7a780d55532c,32955,1731857800648 in 1.8300 sec 2024-11-17T15:38:52,307 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=117, resume processing ppid=116 2024-11-17T15:38:52,307 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, ppid=116, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fdca08c93967309fb9c5dcd33e10e7d5, UNASSIGN in 1.8340 sec 2024-11-17T15:38:52,309 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-11-17T15:38:52,309 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8370 sec 2024-11-17T15:38:52,310 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731857932309"}]},"ts":"1731857932309"} 2024-11-17T15:38:52,311 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-17T15:38:52,311 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-17T15:38:52,312 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8450 sec 2024-11-17T15:38:52,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-17T15:38:52,614 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-17T15:38:52,614 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-17T15:38:52,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] procedure2.ProcedureExecutor(1139): Stored pid=119, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T15:38:52,616 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=119, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T15:38:52,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-17T15:38:52,616 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=119, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T15:38:52,619 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5 2024-11-17T15:38:52,620 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/A, FileablePath, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/B, FileablePath, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/C, FileablePath, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/recovered.edits] 2024-11-17T15:38:52,623 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/A/3665514f4b79422cb7964eda189f49a2 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/A/3665514f4b79422cb7964eda189f49a2 2024-11-17T15:38:52,624 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/A/e323835ddf5a433fbcdf48a4744bcc36 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/A/e323835ddf5a433fbcdf48a4744bcc36 2024-11-17T15:38:52,625 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/A/ea17370150194f7cbb758937addf2db4 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/A/ea17370150194f7cbb758937addf2db4 2024-11-17T15:38:52,627 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/B/5f370a262d0f4994b58140d639b52ed2 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/B/5f370a262d0f4994b58140d639b52ed2 2024-11-17T15:38:52,628 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/B/ec53ba73b00d4a1fb0c651768a09c1e5 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/B/ec53ba73b00d4a1fb0c651768a09c1e5 2024-11-17T15:38:52,628 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/B/f4d73f2d3698425d9d2fa2a5fd13711d to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/B/f4d73f2d3698425d9d2fa2a5fd13711d 2024-11-17T15:38:52,630 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/C/7037bda766474747aada5fc95e34e90b to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/C/7037bda766474747aada5fc95e34e90b 2024-11-17T15:38:52,631 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/C/a5ddf924f964453d8e73d25e7ae1b918 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/C/a5ddf924f964453d8e73d25e7ae1b918 2024-11-17T15:38:52,631 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/C/ca75cd8bedf647c99e87ee04a1c2e409 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/C/ca75cd8bedf647c99e87ee04a1c2e409 2024-11-17T15:38:52,633 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/recovered.edits/301.seqid to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5/recovered.edits/301.seqid 2024-11-17T15:38:52,634 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/fdca08c93967309fb9c5dcd33e10e7d5 2024-11-17T15:38:52,634 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-17T15:38:52,634 DEBUG [PEWorker-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-17T15:38:52,635 DEBUG [PEWorker-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-17T15:38:52,637 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117116b1019859a44448f1d1624c08ac280_fdca08c93967309fb9c5dcd33e10e7d5 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117116b1019859a44448f1d1624c08ac280_fdca08c93967309fb9c5dcd33e10e7d5 2024-11-17T15:38:52,638 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117261771ac4b7040978e756f0bc2b3f4ac_fdca08c93967309fb9c5dcd33e10e7d5 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117261771ac4b7040978e756f0bc2b3f4ac_fdca08c93967309fb9c5dcd33e10e7d5 2024-11-17T15:38:52,638 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411172c704482f45a43d5b639ab13d6cbd64b_fdca08c93967309fb9c5dcd33e10e7d5 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411172c704482f45a43d5b639ab13d6cbd64b_fdca08c93967309fb9c5dcd33e10e7d5 2024-11-17T15:38:52,639 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411174773921fc99941b6b7feafa434ee3809_fdca08c93967309fb9c5dcd33e10e7d5 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411174773921fc99941b6b7feafa434ee3809_fdca08c93967309fb9c5dcd33e10e7d5 2024-11-17T15:38:52,640 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411174f5f6e9ba339407ba367fa5217c806af_fdca08c93967309fb9c5dcd33e10e7d5 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411174f5f6e9ba339407ba367fa5217c806af_fdca08c93967309fb9c5dcd33e10e7d5 2024-11-17T15:38:52,641 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411177d7c0cabdb7249bb87af25dd8dcfbdc2_fdca08c93967309fb9c5dcd33e10e7d5 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411177d7c0cabdb7249bb87af25dd8dcfbdc2_fdca08c93967309fb9c5dcd33e10e7d5 2024-11-17T15:38:52,642 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411178d420cfce30b4d9abdc802491d253b8d_fdca08c93967309fb9c5dcd33e10e7d5 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411178d420cfce30b4d9abdc802491d253b8d_fdca08c93967309fb9c5dcd33e10e7d5 2024-11-17T15:38:52,642 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117968eec3471234322a3f5a35dc4399c59_fdca08c93967309fb9c5dcd33e10e7d5 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117968eec3471234322a3f5a35dc4399c59_fdca08c93967309fb9c5dcd33e10e7d5 2024-11-17T15:38:52,643 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117b2263032a0a849ceb61e72f87c69c54a_fdca08c93967309fb9c5dcd33e10e7d5 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117b2263032a0a849ceb61e72f87c69c54a_fdca08c93967309fb9c5dcd33e10e7d5 2024-11-17T15:38:52,644 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117be95fd6526c34c14b26380d12649db94_fdca08c93967309fb9c5dcd33e10e7d5 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117be95fd6526c34c14b26380d12649db94_fdca08c93967309fb9c5dcd33e10e7d5 2024-11-17T15:38:52,645 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117c879c52bfdac470e8ffce11604a7989a_fdca08c93967309fb9c5dcd33e10e7d5 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117c879c52bfdac470e8ffce11604a7989a_fdca08c93967309fb9c5dcd33e10e7d5 2024-11-17T15:38:52,646 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117d42f43160a9347ef8959d2e12d7b9b17_fdca08c93967309fb9c5dcd33e10e7d5 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117d42f43160a9347ef8959d2e12d7b9b17_fdca08c93967309fb9c5dcd33e10e7d5 2024-11-17T15:38:52,647 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117ef2c1e03af9c49268d81aa9aecd35666_fdca08c93967309fb9c5dcd33e10e7d5 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117ef2c1e03af9c49268d81aa9aecd35666_fdca08c93967309fb9c5dcd33e10e7d5 2024-11-17T15:38:52,647 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117f39cc1839799464b8841930d066c83a8_fdca08c93967309fb9c5dcd33e10e7d5 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117f39cc1839799464b8841930d066c83a8_fdca08c93967309fb9c5dcd33e10e7d5 2024-11-17T15:38:52,648 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117f3daf79a229f4bd7b992b9c2ec427c6e_fdca08c93967309fb9c5dcd33e10e7d5 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117f3daf79a229f4bd7b992b9c2ec427c6e_fdca08c93967309fb9c5dcd33e10e7d5 2024-11-17T15:38:52,649 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117f65c35ffecde42159224b61fb1c2c720_fdca08c93967309fb9c5dcd33e10e7d5 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117f65c35ffecde42159224b61fb1c2c720_fdca08c93967309fb9c5dcd33e10e7d5 2024-11-17T15:38:52,649 DEBUG [PEWorker-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-17T15:38:52,651 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=119, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T15:38:52,653 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-17T15:38:52,654 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-17T15:38:52,655 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=119, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T15:38:52,655 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-17T15:38:52,655 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731857932655"}]},"ts":"9223372036854775807"} 2024-11-17T15:38:52,657 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-17T15:38:52,657 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => fdca08c93967309fb9c5dcd33e10e7d5, NAME => 'TestAcidGuarantees,,1731857902432.fdca08c93967309fb9c5dcd33e10e7d5.', STARTKEY => '', ENDKEY => ''}] 2024-11-17T15:38:52,657 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-17T15:38:52,657 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731857932657"}]},"ts":"9223372036854775807"} 2024-11-17T15:38:52,658 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-17T15:38:52,659 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=119, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T15:38:52,660 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 45 msec 2024-11-17T15:38:52,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-17T15:38:52,723 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-17T15:38:52,723 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-17T15:38:52,732 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobScanAtomicity Thread=241 (was 237) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1842016799_22 at /127.0.0.1:39984 [Waiting for operation #680] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1410560447_22 at /127.0.0.1:40202 [Waiting for operation #630] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-10 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1410560447_22 at /127.0.0.1:49234 [Waiting for operation #552] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_COMPACTED_FILES_DISCHARGER-regionserver/7a780d55532c:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-11 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1842016799_22 at /127.0.0.1:49226 [Waiting for operation #547] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=455 (was 445) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=404 (was 385) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=2902 (was 3119) 2024-11-17T15:38:52,740 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testScanAtomicity Thread=241, OpenFileDescriptor=455, MaxFileDescriptor=1048576, SystemLoadAverage=404, ProcessCount=11, AvailableMemoryMB=2902 2024-11-17T15:38:52,741 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-17T15:38:52,741 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-17T15:38:52,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] procedure2.ProcedureExecutor(1139): Stored pid=120, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-17T15:38:52,743 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=120, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-17T15:38:52,743 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:52,743 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 120 2024-11-17T15:38:52,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=120 2024-11-17T15:38:52,744 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=120, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-17T15:38:52,748 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742265_1441 (size=959) 2024-11-17T15:38:52,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=120 2024-11-17T15:38:53,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=120 2024-11-17T15:38:53,150 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206 2024-11-17T15:38:53,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742266_1442 (size=53) 2024-11-17T15:38:53,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=120 2024-11-17T15:38:53,555 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T15:38:53,556 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 420677de373b14e08d29085670121ff4, disabling compactions & flushes 2024-11-17T15:38:53,556 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. 2024-11-17T15:38:53,556 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. 2024-11-17T15:38:53,556 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. after waiting 0 ms 2024-11-17T15:38:53,556 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. 2024-11-17T15:38:53,556 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. 2024-11-17T15:38:53,556 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 420677de373b14e08d29085670121ff4: Waiting for close lock at 1731857933555Disabling compacts and flushes for region at 1731857933555Disabling writes for close at 1731857933556 (+1 ms)Writing region close event to WAL at 1731857933556Closed at 1731857933556 2024-11-17T15:38:53,557 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=120, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-17T15:38:53,557 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731857933557"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731857933557"}]},"ts":"1731857933557"} 2024-11-17T15:38:53,558 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-17T15:38:53,559 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=120, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-17T15:38:53,559 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731857933559"}]},"ts":"1731857933559"} 2024-11-17T15:38:53,560 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-17T15:38:53,560 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=121, ppid=120, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=420677de373b14e08d29085670121ff4, ASSIGN}] 2024-11-17T15:38:53,561 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=121, ppid=120, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=420677de373b14e08d29085670121ff4, ASSIGN 2024-11-17T15:38:53,562 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=121, ppid=120, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=420677de373b14e08d29085670121ff4, ASSIGN; state=OFFLINE, location=7a780d55532c,32955,1731857800648; forceNewPlan=false, retain=false 2024-11-17T15:38:53,712 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=121 updating hbase:meta row=420677de373b14e08d29085670121ff4, regionState=OPENING, regionLocation=7a780d55532c,32955,1731857800648 2024-11-17T15:38:53,713 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=121, ppid=120, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=420677de373b14e08d29085670121ff4, ASSIGN because future has completed 2024-11-17T15:38:53,714 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; OpenRegionProcedure 420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648}] 2024-11-17T15:38:53,868 INFO [RS_OPEN_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. 2024-11-17T15:38:53,868 DEBUG [RS_OPEN_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(7752): Opening region: {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} 2024-11-17T15:38:53,869 DEBUG [RS_OPEN_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 420677de373b14e08d29085670121ff4 2024-11-17T15:38:53,869 DEBUG [RS_OPEN_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T15:38:53,869 DEBUG [RS_OPEN_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(7794): checking encryption for 420677de373b14e08d29085670121ff4 2024-11-17T15:38:53,869 DEBUG [RS_OPEN_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(7797): checking classloading for 420677de373b14e08d29085670121ff4 2024-11-17T15:38:53,870 INFO [StoreOpener-420677de373b14e08d29085670121ff4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 420677de373b14e08d29085670121ff4 2024-11-17T15:38:53,871 INFO [StoreOpener-420677de373b14e08d29085670121ff4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 420677de373b14e08d29085670121ff4 columnFamilyName A 2024-11-17T15:38:53,871 DEBUG [StoreOpener-420677de373b14e08d29085670121ff4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:53,872 INFO [StoreOpener-420677de373b14e08d29085670121ff4-1 {}] regionserver.HStore(327): Store=420677de373b14e08d29085670121ff4/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T15:38:53,872 INFO [StoreOpener-420677de373b14e08d29085670121ff4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 420677de373b14e08d29085670121ff4 2024-11-17T15:38:53,873 INFO [StoreOpener-420677de373b14e08d29085670121ff4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 420677de373b14e08d29085670121ff4 columnFamilyName B 2024-11-17T15:38:53,873 DEBUG [StoreOpener-420677de373b14e08d29085670121ff4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:53,873 INFO [StoreOpener-420677de373b14e08d29085670121ff4-1 {}] regionserver.HStore(327): Store=420677de373b14e08d29085670121ff4/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T15:38:53,873 INFO [StoreOpener-420677de373b14e08d29085670121ff4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 420677de373b14e08d29085670121ff4 2024-11-17T15:38:53,874 INFO [StoreOpener-420677de373b14e08d29085670121ff4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 420677de373b14e08d29085670121ff4 columnFamilyName C 2024-11-17T15:38:53,874 DEBUG [StoreOpener-420677de373b14e08d29085670121ff4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:38:53,874 INFO [StoreOpener-420677de373b14e08d29085670121ff4-1 {}] regionserver.HStore(327): Store=420677de373b14e08d29085670121ff4/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T15:38:53,874 DEBUG [RS_OPEN_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(1038): replaying wal for 420677de373b14e08d29085670121ff4 2024-11-17T15:38:53,875 DEBUG [RS_OPEN_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4 2024-11-17T15:38:53,875 DEBUG [RS_OPEN_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4 2024-11-17T15:38:53,876 DEBUG [RS_OPEN_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(1048): stopping wal replay for 420677de373b14e08d29085670121ff4 2024-11-17T15:38:53,876 DEBUG [RS_OPEN_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(1060): Cleaning up temporary data for 420677de373b14e08d29085670121ff4 2024-11-17T15:38:53,876 DEBUG [RS_OPEN_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-17T15:38:53,877 DEBUG [RS_OPEN_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(1093): writing seq id for 420677de373b14e08d29085670121ff4 2024-11-17T15:38:53,878 DEBUG [RS_OPEN_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-17T15:38:53,879 INFO [RS_OPEN_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(1114): Opened 420677de373b14e08d29085670121ff4; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67633297, jitterRate=0.007814660668373108}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-17T15:38:53,879 DEBUG [RS_OPEN_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 420677de373b14e08d29085670121ff4 2024-11-17T15:38:53,879 DEBUG [RS_OPEN_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(1006): Region open journal for 420677de373b14e08d29085670121ff4: Running coprocessor pre-open hook at 1731857933869Writing region info on filesystem at 1731857933869Initializing all the Stores at 1731857933870 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731857933870Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731857933870Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731857933870Cleaning up temporary data from old regions at 1731857933876 (+6 ms)Running coprocessor post-open hooks at 1731857933879 (+3 ms)Region opened successfully at 1731857933879 2024-11-17T15:38:53,880 INFO [RS_OPEN_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., pid=122, masterSystemTime=1731857933865 2024-11-17T15:38:53,881 DEBUG [RS_OPEN_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. 2024-11-17T15:38:53,881 INFO [RS_OPEN_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. 2024-11-17T15:38:53,882 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=121 updating hbase:meta row=420677de373b14e08d29085670121ff4, regionState=OPEN, openSeqNum=2, regionLocation=7a780d55532c,32955,1731857800648 2024-11-17T15:38:53,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=120 2024-11-17T15:38:53,883 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=122, ppid=121, state=RUNNABLE, hasLock=false; OpenRegionProcedure 420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 because future has completed 2024-11-17T15:38:53,885 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-11-17T15:38:53,885 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; OpenRegionProcedure 420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 in 169 msec 2024-11-17T15:38:53,886 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=121, resume processing ppid=120 2024-11-17T15:38:53,886 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, ppid=120, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=420677de373b14e08d29085670121ff4, ASSIGN in 325 msec 2024-11-17T15:38:53,886 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=120, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-17T15:38:53,887 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731857933886"}]},"ts":"1731857933886"} 2024-11-17T15:38:53,888 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-17T15:38:53,888 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=120, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-17T15:38:53,889 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1470 sec 2024-11-17T15:38:54,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=120 2024-11-17T15:38:54,893 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-17T15:38:54,894 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28ad78a3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:38:54,894 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7a780d55532c,44223,-1 for getting cluster id 2024-11-17T15:38:54,894 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T15:38:54,895 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e01f4cd2-8971-4a73-861a-2ca0deb4f76e' 2024-11-17T15:38:54,895 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T15:38:54,895 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e01f4cd2-8971-4a73-861a-2ca0deb4f76e" 2024-11-17T15:38:54,895 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ae8441e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:38:54,895 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7a780d55532c,44223,-1] 2024-11-17T15:38:54,896 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T15:38:54,896 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:38:54,897 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57198, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T15:38:54,897 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@119d554a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:38:54,897 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T15:38:54,898 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7a780d55532c,32955,1731857800648, seqNum=-1] 2024-11-17T15:38:54,898 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T15:38:54,899 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48374, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T15:38:54,899 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-17T15:38:54,900 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is 7a780d55532c,44223,1731857799978 2024-11-17T15:38:54,900 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@26bda058 2024-11-17T15:38:54,900 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-17T15:38:54,901 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57202, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-17T15:38:54,902 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ac69b7f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:38:54,902 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7a780d55532c,44223,-1 for getting cluster id 2024-11-17T15:38:54,902 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T15:38:54,903 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e01f4cd2-8971-4a73-861a-2ca0deb4f76e' 2024-11-17T15:38:54,903 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T15:38:54,903 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e01f4cd2-8971-4a73-861a-2ca0deb4f76e" 2024-11-17T15:38:54,903 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c3546ba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:38:54,903 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7a780d55532c,44223,-1] 2024-11-17T15:38:54,903 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T15:38:54,903 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:38:54,904 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57222, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T15:38:54,904 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14b10a7d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:38:54,904 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@277a5c68, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:38:54,905 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7a780d55532c,44223,-1 for getting cluster id 2024-11-17T15:38:54,905 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T15:38:54,905 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e01f4cd2-8971-4a73-861a-2ca0deb4f76e' 2024-11-17T15:38:54,905 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T15:38:54,905 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e01f4cd2-8971-4a73-861a-2ca0deb4f76e" 2024-11-17T15:38:54,906 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e4fa942, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:38:54,906 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7a780d55532c,44223,-1] 2024-11-17T15:38:54,906 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T15:38:54,906 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:38:54,906 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57240, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T15:38:54,907 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e176529, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:38:54,907 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1df5cdc7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:38:54,907 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7a780d55532c,44223,-1 for getting cluster id 2024-11-17T15:38:54,907 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T15:38:54,908 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e01f4cd2-8971-4a73-861a-2ca0deb4f76e' 2024-11-17T15:38:54,908 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T15:38:54,908 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e01f4cd2-8971-4a73-861a-2ca0deb4f76e" 2024-11-17T15:38:54,908 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@780defbb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:38:54,908 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7a780d55532c,44223,-1] 2024-11-17T15:38:54,908 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T15:38:54,908 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:38:54,909 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57260, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T15:38:54,909 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@decb360, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:38:54,910 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22b9b902, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:38:54,910 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7a780d55532c,44223,-1 for getting cluster id 2024-11-17T15:38:54,910 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T15:38:54,910 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e01f4cd2-8971-4a73-861a-2ca0deb4f76e' 2024-11-17T15:38:54,910 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T15:38:54,910 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e01f4cd2-8971-4a73-861a-2ca0deb4f76e" 2024-11-17T15:38:54,911 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e3f4c5d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:38:54,911 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7a780d55532c,44223,-1] 2024-11-17T15:38:54,911 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T15:38:54,911 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:38:54,911 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57286, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T15:38:54,912 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@713b4ed0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:38:54,912 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ff7674f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:38:54,912 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7a780d55532c,44223,-1 for getting cluster id 2024-11-17T15:38:54,912 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T15:38:54,913 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e01f4cd2-8971-4a73-861a-2ca0deb4f76e' 2024-11-17T15:38:54,913 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T15:38:54,913 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e01f4cd2-8971-4a73-861a-2ca0deb4f76e" 2024-11-17T15:38:54,913 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a6d8a77, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:38:54,913 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7a780d55532c,44223,-1] 2024-11-17T15:38:54,913 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T15:38:54,913 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:38:54,914 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57310, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T15:38:54,914 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d37d209, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:38:54,915 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@107467c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:38:54,915 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7a780d55532c,44223,-1 for getting cluster id 2024-11-17T15:38:54,915 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T15:38:54,915 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e01f4cd2-8971-4a73-861a-2ca0deb4f76e' 2024-11-17T15:38:54,915 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T15:38:54,915 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e01f4cd2-8971-4a73-861a-2ca0deb4f76e" 2024-11-17T15:38:54,916 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@416a8470, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:38:54,916 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7a780d55532c,44223,-1] 2024-11-17T15:38:54,916 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T15:38:54,916 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:38:54,916 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57334, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T15:38:54,917 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c318b82, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:38:54,917 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2acfd5e4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:38:54,918 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7a780d55532c,44223,-1 for getting cluster id 2024-11-17T15:38:54,918 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T15:38:54,918 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e01f4cd2-8971-4a73-861a-2ca0deb4f76e' 2024-11-17T15:38:54,918 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T15:38:54,918 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e01f4cd2-8971-4a73-861a-2ca0deb4f76e" 2024-11-17T15:38:54,918 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6bdf53b5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:38:54,918 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7a780d55532c,44223,-1] 2024-11-17T15:38:54,918 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T15:38:54,919 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:38:54,919 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57356, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T15:38:54,919 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52af0149, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:38:54,920 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@379767c9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:38:54,920 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7a780d55532c,44223,-1 for getting cluster id 2024-11-17T15:38:54,920 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T15:38:54,920 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e01f4cd2-8971-4a73-861a-2ca0deb4f76e' 2024-11-17T15:38:54,921 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T15:38:54,921 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e01f4cd2-8971-4a73-861a-2ca0deb4f76e" 2024-11-17T15:38:54,921 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4730fdf3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:38:54,921 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7a780d55532c,44223,-1] 2024-11-17T15:38:54,921 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T15:38:54,921 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:38:54,921 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57378, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T15:38:54,922 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4dfb0296, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:38:54,922 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69f15306, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:38:54,922 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7a780d55532c,44223,-1 for getting cluster id 2024-11-17T15:38:54,922 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T15:38:54,923 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e01f4cd2-8971-4a73-861a-2ca0deb4f76e' 2024-11-17T15:38:54,923 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T15:38:54,923 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e01f4cd2-8971-4a73-861a-2ca0deb4f76e" 2024-11-17T15:38:54,923 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41c6a46, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:38:54,923 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7a780d55532c,44223,-1] 2024-11-17T15:38:54,923 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T15:38:54,924 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:38:54,924 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57384, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T15:38:54,924 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30b7ebd3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:38:54,925 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4df5a2b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:38:54,925 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7a780d55532c,44223,-1 for getting cluster id 2024-11-17T15:38:54,925 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T15:38:54,926 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e01f4cd2-8971-4a73-861a-2ca0deb4f76e' 2024-11-17T15:38:54,926 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T15:38:54,926 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e01f4cd2-8971-4a73-861a-2ca0deb4f76e" 2024-11-17T15:38:54,926 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@17f71f69, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:38:54,926 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7a780d55532c,44223,-1] 2024-11-17T15:38:54,927 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T15:38:54,927 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:38:54,927 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57394, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T15:38:54,928 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@644b4d14, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:38:54,928 DEBUG [Thread-2003 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T15:38:54,928 DEBUG [Thread-1992 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T15:38:54,928 DEBUG [Thread-1996 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T15:38:54,928 DEBUG [Thread-1998 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T15:38:54,928 DEBUG [Thread-2009 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T15:38:54,929 DEBUG [Thread-2007 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T15:38:54,929 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7a780d55532c,32955,1731857800648, seqNum=-1] 2024-11-17T15:38:54,929 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7a780d55532c,32955,1731857800648, seqNum=-1] 2024-11-17T15:38:54,929 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7a780d55532c,32955,1731857800648, seqNum=-1] 2024-11-17T15:38:54,929 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T15:38:54,929 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T15:38:54,929 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T15:38:54,929 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7a780d55532c,32955,1731857800648, seqNum=-1] 2024-11-17T15:38:54,929 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T15:38:54,929 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7a780d55532c,32955,1731857800648, seqNum=-1] 2024-11-17T15:38:54,930 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T15:38:54,930 DEBUG [Thread-2005 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T15:38:54,930 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48390, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T15:38:54,930 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48398, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T15:38:54,930 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48414, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T15:38:54,931 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48430, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T15:38:54,931 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48406, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T15:38:54,931 DEBUG [Thread-1994 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T15:38:54,931 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7a780d55532c,32955,1731857800648, seqNum=-1] 2024-11-17T15:38:54,931 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7a780d55532c,32955,1731857800648, seqNum=-1] 2024-11-17T15:38:54,931 DEBUG [Thread-1990 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T15:38:54,931 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T15:38:54,931 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T15:38:54,932 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7a780d55532c,32955,1731857800648, seqNum=-1] 2024-11-17T15:38:54,932 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T15:38:54,932 DEBUG [Thread-2001 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T15:38:54,932 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48442, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T15:38:54,932 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2] 2024-11-17T15:38:54,933 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2] 2024-11-17T15:38:54,933 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48458, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T15:38:54,933 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48446, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T15:38:54,933 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2] 2024-11-17T15:38:54,934 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2] 2024-11-17T15:38:54,934 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2] 2024-11-17T15:38:54,934 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7a780d55532c,32955,1731857800648, seqNum=-1] 2024-11-17T15:38:54,934 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T15:38:54,935 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7a780d55532c,32955,1731857800648, seqNum=-1] 2024-11-17T15:38:54,935 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2] 2024-11-17T15:38:54,935 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T15:38:54,935 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2] 2024-11-17T15:38:54,935 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48462, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T15:38:54,936 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2] 2024-11-17T15:38:54,936 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48474, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T15:38:54,936 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T15:38:54,937 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2] 2024-11-17T15:38:54,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees 2024-11-17T15:38:54,937 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2] 2024-11-17T15:38:54,938 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T15:38:54,938 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T15:38:54,938 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T15:38:54,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-17T15:38:54,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(8855): Flush requested on 420677de373b14e08d29085670121ff4 2024-11-17T15:38:54,944 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T15:38:54,945 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 420677de373b14e08d29085670121ff4 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-17T15:38:54,971 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/A/fac8538d881540e09b34eccbbbd560a5 is 50, key is test_row_0/A:col10/1731857934942/Put/seqid=0 2024-11-17T15:38:54,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742267_1443 (size=16681) 2024-11-17T15:38:55,025 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:55,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48474 deadline: 1731857995013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:55,026 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:55,026 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:55,026 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:55,026 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:38:55,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48458 deadline: 1731857995021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:55,026 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:55,026 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:55,026 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:38:55,032 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:55,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48406 deadline: 1731857995022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:55,032 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:55,032 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:55,032 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:38:55,032 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:55,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48390 deadline: 1731857995024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:55,033 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:55,033 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:55,033 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:38:55,037 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:55,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48442 deadline: 1731857995026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:55,038 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:55,038 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:55,038 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:38:55,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-17T15:38:55,090 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-17T15:38:55,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. 2024-11-17T15:38:55,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. as already flushing 2024-11-17T15:38:55,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. 2024-11-17T15:38:55,091 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:55,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:55,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:55,139 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:55,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48474 deadline: 1731857995132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:55,140 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:55,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48458 deadline: 1731857995133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:55,140 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:55,140 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:55,140 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:38:55,140 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:55,140 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:55,140 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:38:55,150 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:55,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48406 deadline: 1731857995143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:55,150 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:55,150 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:55,151 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:55,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48390 deadline: 1731857995143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:55,151 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:38:55,151 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:55,151 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:55,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48442 deadline: 1731857995143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:55,151 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:55,151 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:38:55,151 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:55,151 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:55,151 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:38:55,244 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-17T15:38:55,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. 2024-11-17T15:38:55,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. as already flushing 2024-11-17T15:38:55,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. 2024-11-17T15:38:55,244 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:55,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:55,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:55,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-17T15:38:55,346 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:55,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48474 deadline: 1731857995342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:55,346 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:55,347 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:55,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48458 deadline: 1731857995343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:55,347 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:55,347 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:38:55,347 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:55,347 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:55,347 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:38:55,356 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:55,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48406 deadline: 1731857995353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:55,356 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:55,356 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:55,356 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:55,356 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:38:55,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48442 deadline: 1731857995353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:55,356 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:55,357 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:55,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48390 deadline: 1731857995353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:55,357 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:55,357 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:38:55,357 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:55,357 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:55,357 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:38:55,375 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/A/fac8538d881540e09b34eccbbbd560a5 2024-11-17T15:38:55,395 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/B/1634d6811e2e49e7bcbccf469dc01b0f is 50, key is test_row_0/B:col10/1731857934942/Put/seqid=0 2024-11-17T15:38:55,397 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-17T15:38:55,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. 2024-11-17T15:38:55,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. as already flushing 2024-11-17T15:38:55,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. 2024-11-17T15:38:55,397 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:55,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:55,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:55,399 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742268_1444 (size=12001) 2024-11-17T15:38:55,549 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-17T15:38:55,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. 2024-11-17T15:38:55,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. as already flushing 2024-11-17T15:38:55,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. 2024-11-17T15:38:55,550 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:55,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:55,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:55,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-17T15:38:55,659 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:55,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48474 deadline: 1731857995653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:55,659 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:55,659 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:55,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48458 deadline: 1731857995653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:55,659 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:55,659 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:38:55,659 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:55,659 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:55,659 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:38:55,669 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:55,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48406 deadline: 1731857995663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:55,669 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:55,670 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:55,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48442 deadline: 1731857995663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:55,670 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:55,670 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:38:55,670 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:55,670 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:55,670 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:55,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48390 deadline: 1731857995663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:55,670 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:38:55,670 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:55,670 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:55,670 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:38:55,702 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-17T15:38:55,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. 2024-11-17T15:38:55,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. as already flushing 2024-11-17T15:38:55,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. 2024-11-17T15:38:55,703 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:55,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:55,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:55,800 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/B/1634d6811e2e49e7bcbccf469dc01b0f 2024-11-17T15:38:55,824 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/C/49fd28d38a584da58dcd21d8858e73d1 is 50, key is test_row_0/C:col10/1731857934942/Put/seqid=0 2024-11-17T15:38:55,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742269_1445 (size=12001) 2024-11-17T15:38:55,855 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-17T15:38:55,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. 2024-11-17T15:38:55,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. as already flushing 2024-11-17T15:38:55,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. 2024-11-17T15:38:55,856 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:55,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:55,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:56,008 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-17T15:38:56,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. 2024-11-17T15:38:56,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. as already flushing 2024-11-17T15:38:56,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. 2024-11-17T15:38:56,009 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:56,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:56,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:56,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-17T15:38:56,161 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-17T15:38:56,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. 2024-11-17T15:38:56,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. as already flushing 2024-11-17T15:38:56,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. 2024-11-17T15:38:56,161 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:56,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:56,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:56,168 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:56,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48474 deadline: 1731857996163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:56,169 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:56,169 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:56,169 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:38:56,179 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:56,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48458 deadline: 1731857996173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:56,179 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:56,179 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:56,179 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:56,179 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:38:56,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48406 deadline: 1731857996173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:56,180 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:56,180 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:56,180 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:56,180 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:38:56,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48390 deadline: 1731857996173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:56,180 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:56,180 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:56,180 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:38:56,190 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:56,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48442 deadline: 1731857996183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:56,191 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:56,191 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:56,191 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:38:56,228 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/C/49fd28d38a584da58dcd21d8858e73d1 2024-11-17T15:38:56,232 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/A/fac8538d881540e09b34eccbbbd560a5 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/A/fac8538d881540e09b34eccbbbd560a5 2024-11-17T15:38:56,235 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/A/fac8538d881540e09b34eccbbbd560a5, entries=250, sequenceid=12, filesize=16.3 K 2024-11-17T15:38:56,236 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/B/1634d6811e2e49e7bcbccf469dc01b0f as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/B/1634d6811e2e49e7bcbccf469dc01b0f 2024-11-17T15:38:56,239 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/B/1634d6811e2e49e7bcbccf469dc01b0f, entries=150, sequenceid=12, filesize=11.7 K 2024-11-17T15:38:56,239 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/C/49fd28d38a584da58dcd21d8858e73d1 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/C/49fd28d38a584da58dcd21d8858e73d1 2024-11-17T15:38:56,243 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/C/49fd28d38a584da58dcd21d8858e73d1, entries=150, sequenceid=12, filesize=11.7 K 2024-11-17T15:38:56,244 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for 420677de373b14e08d29085670121ff4 in 1300ms, sequenceid=12, compaction requested=false 2024-11-17T15:38:56,244 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 420677de373b14e08d29085670121ff4: 2024-11-17T15:38:56,314 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-17T15:38:56,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. 2024-11-17T15:38:56,314 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2902): Flushing 420677de373b14e08d29085670121ff4 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-17T15:38:56,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/A/9c313cd408cd44719b994aeb5213bc54 is 50, key is test_row_0/A:col10/1731857935021/Put/seqid=0 2024-11-17T15:38:56,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742270_1446 (size=12001) 2024-11-17T15:38:56,723 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/A/9c313cd408cd44719b994aeb5213bc54 2024-11-17T15:38:56,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/B/54e7e6c4ca354a46ab46303f28a686c2 is 50, key is test_row_0/B:col10/1731857935021/Put/seqid=0 2024-11-17T15:38:56,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742271_1447 (size=12001) 2024-11-17T15:38:57,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-17T15:38:57,137 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/B/54e7e6c4ca354a46ab46303f28a686c2 2024-11-17T15:38:57,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/C/ba0aafb62dc14027a9872333e1007e04 is 50, key is test_row_0/C:col10/1731857935021/Put/seqid=0 2024-11-17T15:38:57,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742272_1448 (size=12001) 2024-11-17T15:38:57,190 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. as already flushing 2024-11-17T15:38:57,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(8855): Flush requested on 420677de373b14e08d29085670121ff4 2024-11-17T15:38:57,211 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:57,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48406 deadline: 1731857997201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:57,211 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:57,211 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:57,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48390 deadline: 1731857997202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:57,212 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:57,212 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:38:57,212 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:57,212 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:57,212 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:57,212 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:57,212 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:38:57,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48474 deadline: 1731857997204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:57,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48442 deadline: 1731857997203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:57,212 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:57,212 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:57,212 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:38:57,212 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 7a780d55532c,32955,1731857800648 for 'test_row_0' in 420677de373b14e08d29085670121ff4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2186 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T15:38:57,213 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:57,213 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:57,213 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:38:57,216 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:57,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48458 deadline: 1731857997205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:57,216 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:57,216 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:57,216 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:38:57,319 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:57,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48406 deadline: 1731857997313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:57,320 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:57,320 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:57,320 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:38:57,332 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:57,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48390 deadline: 1731857997323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:57,332 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:57,332 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:57,332 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:57,332 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:38:57,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48474 deadline: 1731857997323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:57,332 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:57,332 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:57,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48458 deadline: 1731857997323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:57,332 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:57,332 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:38:57,333 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:57,333 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:57,333 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:38:57,530 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:57,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48406 deadline: 1731857997523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:57,530 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:57,531 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:57,531 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:38:57,549 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/C/ba0aafb62dc14027a9872333e1007e04 2024-11-17T15:38:57,551 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:57,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48390 deadline: 1731857997543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:57,551 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:57,551 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:57,551 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:38:57,553 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:57,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48474 deadline: 1731857997543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:57,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/A/9c313cd408cd44719b994aeb5213bc54 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/A/9c313cd408cd44719b994aeb5213bc54 2024-11-17T15:38:57,553 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:57,553 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:57,553 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:38:57,553 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:57,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48458 deadline: 1731857997543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:57,554 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:57,554 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:57,554 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:38:57,557 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/A/9c313cd408cd44719b994aeb5213bc54, entries=150, sequenceid=38, filesize=11.7 K 2024-11-17T15:38:57,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/B/54e7e6c4ca354a46ab46303f28a686c2 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/B/54e7e6c4ca354a46ab46303f28a686c2 2024-11-17T15:38:57,560 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/B/54e7e6c4ca354a46ab46303f28a686c2, entries=150, sequenceid=38, filesize=11.7 K 2024-11-17T15:38:57,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/C/ba0aafb62dc14027a9872333e1007e04 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/C/ba0aafb62dc14027a9872333e1007e04 2024-11-17T15:38:57,564 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/C/ba0aafb62dc14027a9872333e1007e04, entries=150, sequenceid=38, filesize=11.7 K 2024-11-17T15:38:57,565 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 420677de373b14e08d29085670121ff4 in 1251ms, sequenceid=38, compaction requested=false 2024-11-17T15:38:57,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2603): Flush status journal for 420677de373b14e08d29085670121ff4: 2024-11-17T15:38:57,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. 2024-11-17T15:38:57,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=124 2024-11-17T15:38:57,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.HMaster(4169): Remote procedure done, pid=124 2024-11-17T15:38:57,568 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-11-17T15:38:57,568 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6280 sec 2024-11-17T15:38:57,569 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees in 2.6320 sec 2024-11-17T15:38:57,817 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-17T15:38:57,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(8855): Flush requested on 420677de373b14e08d29085670121ff4 2024-11-17T15:38:57,836 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T15:38:57,836 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 420677de373b14e08d29085670121ff4 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-17T15:38:57,839 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/A/91d6f5810c9049b3862101472bea8900 is 50, key is test_row_0/A:col10/1731857937199/Put/seqid=0 2024-11-17T15:38:57,845 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742273_1449 (size=14341) 2024-11-17T15:38:57,846 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/A/91d6f5810c9049b3862101472bea8900 2024-11-17T15:38:57,853 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/B/d029d99f5dc94fb69fb270c6feeb1f7d is 50, key is test_row_0/B:col10/1731857937199/Put/seqid=0 2024-11-17T15:38:57,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742274_1450 (size=12001) 2024-11-17T15:38:57,919 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:57,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48458 deadline: 1731857997911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:57,919 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:57,920 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:57,920 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:38:57,929 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:57,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48406 deadline: 1731857997917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:57,929 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:57,929 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:57,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48390 deadline: 1731857997918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:57,929 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:57,930 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:38:57,930 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:57,930 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:57,930 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:38:57,931 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:57,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48474 deadline: 1731857997919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:57,931 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:57,931 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:57,932 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:38:58,032 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:58,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48458 deadline: 1731857998023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:58,032 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:58,032 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:58,032 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:38:58,036 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:58,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48406 deadline: 1731857998033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:58,036 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:58,036 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:58,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48390 deadline: 1731857998033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:58,036 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:58,036 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:38:58,037 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:58,037 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:58,037 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:38:58,048 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:58,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48474 deadline: 1731857998043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:58,048 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:58,048 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:58,048 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:38:58,251 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:58,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48458 deadline: 1731857998243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:58,252 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:58,252 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:58,252 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:58,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48406 deadline: 1731857998244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:58,252 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:38:58,252 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:58,252 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:58,252 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:58,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48390 deadline: 1731857998244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:58,252 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:38:58,252 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:58,252 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:58,252 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:38:58,256 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/B/d029d99f5dc94fb69fb270c6feeb1f7d 2024-11-17T15:38:58,261 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:58,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48474 deadline: 1731857998253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:58,261 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:58,261 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:58,261 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:38:58,263 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/C/49277a44575845a98e31f8543f9639b5 is 50, key is test_row_0/C:col10/1731857937199/Put/seqid=0 2024-11-17T15:38:58,275 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742275_1451 (size=12001) 2024-11-17T15:38:58,573 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:58,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48458 deadline: 1731857998563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:58,573 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:58,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48406 deadline: 1731857998563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:58,573 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:58,573 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:58,574 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:38:58,574 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:58,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48390 deadline: 1731857998563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:58,574 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:58,574 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:58,574 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:38:58,574 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:58,574 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:58,574 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:38:58,576 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:58,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48474 deadline: 1731857998573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:58,577 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:58,577 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:58,577 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:38:58,676 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/C/49277a44575845a98e31f8543f9639b5 2024-11-17T15:38:58,680 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/A/91d6f5810c9049b3862101472bea8900 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/A/91d6f5810c9049b3862101472bea8900 2024-11-17T15:38:58,683 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/A/91d6f5810c9049b3862101472bea8900, entries=200, sequenceid=49, filesize=14.0 K 2024-11-17T15:38:58,683 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/B/d029d99f5dc94fb69fb270c6feeb1f7d as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/B/d029d99f5dc94fb69fb270c6feeb1f7d 2024-11-17T15:38:58,686 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/B/d029d99f5dc94fb69fb270c6feeb1f7d, entries=150, sequenceid=49, filesize=11.7 K 2024-11-17T15:38:58,687 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/C/49277a44575845a98e31f8543f9639b5 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/C/49277a44575845a98e31f8543f9639b5 2024-11-17T15:38:58,690 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/C/49277a44575845a98e31f8543f9639b5, entries=150, sequenceid=49, filesize=11.7 K 2024-11-17T15:38:58,690 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 420677de373b14e08d29085670121ff4 in 854ms, sequenceid=49, compaction requested=true 2024-11-17T15:38:58,691 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 420677de373b14e08d29085670121ff4: 2024-11-17T15:38:58,691 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 420677de373b14e08d29085670121ff4:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T15:38:58,691 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T15:38:58,691 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 420677de373b14e08d29085670121ff4:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T15:38:58,691 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T15:38:58,691 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T15:38:58,691 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 420677de373b14e08d29085670121ff4:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T15:38:58,691 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T15:38:58,691 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T15:38:58,692 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 43023 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T15:38:58,692 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1541): 420677de373b14e08d29085670121ff4/A is initiating minor compaction (all files) 2024-11-17T15:38:58,692 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 420677de373b14e08d29085670121ff4/A in TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. 2024-11-17T15:38:58,692 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/A/fac8538d881540e09b34eccbbbd560a5, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/A/9c313cd408cd44719b994aeb5213bc54, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/A/91d6f5810c9049b3862101472bea8900] into tmpdir=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp, totalSize=42.0 K 2024-11-17T15:38:58,692 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T15:38:58,692 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1541): 420677de373b14e08d29085670121ff4/B is initiating minor compaction (all files) 2024-11-17T15:38:58,692 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 420677de373b14e08d29085670121ff4/B in TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. 2024-11-17T15:38:58,692 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/B/1634d6811e2e49e7bcbccf469dc01b0f, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/B/54e7e6c4ca354a46ab46303f28a686c2, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/B/d029d99f5dc94fb69fb270c6feeb1f7d] into tmpdir=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp, totalSize=35.2 K 2024-11-17T15:38:58,693 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting fac8538d881540e09b34eccbbbd560a5, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731857934935 2024-11-17T15:38:58,693 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting 1634d6811e2e49e7bcbccf469dc01b0f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731857934941 2024-11-17T15:38:58,693 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9c313cd408cd44719b994aeb5213bc54, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1731857935020 2024-11-17T15:38:58,693 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting 54e7e6c4ca354a46ab46303f28a686c2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1731857935020 2024-11-17T15:38:58,693 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 91d6f5810c9049b3862101472bea8900, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731857937199 2024-11-17T15:38:58,693 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting d029d99f5dc94fb69fb270c6feeb1f7d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731857937199 2024-11-17T15:38:58,699 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 420677de373b14e08d29085670121ff4#A#compaction#374 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T15:38:58,699 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/A/bb8bbbc8baae426890539d5d7bed9c5f is 50, key is test_row_0/A:col10/1731857937199/Put/seqid=0 2024-11-17T15:38:58,700 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 420677de373b14e08d29085670121ff4#B#compaction#375 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T15:38:58,701 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/B/83e66da322354814add77a9da6c7c038 is 50, key is test_row_0/B:col10/1731857937199/Put/seqid=0 2024-11-17T15:38:58,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742276_1452 (size=12104) 2024-11-17T15:38:58,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742277_1453 (size=12104) 2024-11-17T15:38:59,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(8855): Flush requested on 420677de373b14e08d29085670121ff4 2024-11-17T15:38:59,090 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T15:38:59,090 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 420677de373b14e08d29085670121ff4 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-17T15:38:59,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-17T15:38:59,093 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T15:38:59,094 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T15:38:59,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees 2024-11-17T15:38:59,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-17T15:38:59,095 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T15:38:59,096 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T15:38:59,096 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T15:38:59,096 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/A/e60042d919a04568be5b1728416beef3 is 50, key is test_row_0/A:col10/1731857939090/Put/seqid=0 2024-11-17T15:38:59,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742278_1454 (size=12001) 2024-11-17T15:38:59,105 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:59,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48458 deadline: 1731857999101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:59,106 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:59,106 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:59,106 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:38:59,110 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:59,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48406 deadline: 1731857999102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:59,111 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:59,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48390 deadline: 1731857999104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:59,112 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:59,112 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:59,112 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:38:59,112 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:59,112 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:59,112 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:38:59,113 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/A/bb8bbbc8baae426890539d5d7bed9c5f as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/A/bb8bbbc8baae426890539d5d7bed9c5f 2024-11-17T15:38:59,113 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/B/83e66da322354814add77a9da6c7c038 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/B/83e66da322354814add77a9da6c7c038 2024-11-17T15:38:59,118 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 420677de373b14e08d29085670121ff4/A of 420677de373b14e08d29085670121ff4 into bb8bbbc8baae426890539d5d7bed9c5f(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T15:38:59,118 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 420677de373b14e08d29085670121ff4: 2024-11-17T15:38:59,118 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., storeName=420677de373b14e08d29085670121ff4/A, priority=13, startTime=1731857938691; duration=0sec 2024-11-17T15:38:59,118 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 420677de373b14e08d29085670121ff4/B of 420677de373b14e08d29085670121ff4 into 83e66da322354814add77a9da6c7c038(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T15:38:59,118 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T15:38:59,118 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 420677de373b14e08d29085670121ff4:A 2024-11-17T15:38:59,118 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 420677de373b14e08d29085670121ff4: 2024-11-17T15:38:59,118 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., storeName=420677de373b14e08d29085670121ff4/B, priority=13, startTime=1731857938691; duration=0sec 2024-11-17T15:38:59,118 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T15:38:59,118 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T15:38:59,118 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 420677de373b14e08d29085670121ff4:B 2024-11-17T15:38:59,119 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T15:38:59,119 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:59,119 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1541): 420677de373b14e08d29085670121ff4/C is initiating minor compaction (all files) 2024-11-17T15:38:59,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48474 deadline: 1731857999105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:59,119 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 420677de373b14e08d29085670121ff4/C in TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. 2024-11-17T15:38:59,119 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/C/49fd28d38a584da58dcd21d8858e73d1, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/C/ba0aafb62dc14027a9872333e1007e04, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/C/49277a44575845a98e31f8543f9639b5] into tmpdir=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp, totalSize=35.2 K 2024-11-17T15:38:59,120 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 49fd28d38a584da58dcd21d8858e73d1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731857934941 2024-11-17T15:38:59,120 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting ba0aafb62dc14027a9872333e1007e04, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1731857935020 2024-11-17T15:38:59,120 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 49277a44575845a98e31f8543f9639b5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731857937199 2024-11-17T15:38:59,121 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:59,121 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:59,121 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:38:59,128 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 420677de373b14e08d29085670121ff4#C#compaction#377 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T15:38:59,129 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/C/3e5a1aef52334141ac642133c0432a17 is 50, key is test_row_0/C:col10/1731857937199/Put/seqid=0 2024-11-17T15:38:59,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742279_1455 (size=12104) 2024-11-17T15:38:59,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-17T15:38:59,218 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:59,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48458 deadline: 1731857999213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:59,219 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:59,219 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:59,219 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:38:59,230 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:59,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48406 deadline: 1731857999223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:59,230 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:59,230 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:59,230 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:59,230 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:38:59,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48390 deadline: 1731857999223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:59,231 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:59,231 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:59,231 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:59,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48474 deadline: 1731857999224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:59,231 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:38:59,231 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:59,231 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:59,231 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:38:59,242 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:59,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48442 deadline: 1731857999233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:59,243 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 7a780d55532c,32955,1731857800648 for 'test_row_0' in 420677de373b14e08d29085670121ff4 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4217 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T15:38:59,243 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:59,243 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:59,243 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:38:59,248 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-17T15:38:59,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. 2024-11-17T15:38:59,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. as already flushing 2024-11-17T15:38:59,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. 2024-11-17T15:38:59,248 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:59,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:59,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:59,401 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-17T15:38:59,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. 2024-11-17T15:38:59,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. as already flushing 2024-11-17T15:38:59,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. 2024-11-17T15:38:59,401 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:59,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:59,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:59,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-17T15:38:59,430 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:59,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48458 deadline: 1731857999423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:59,431 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:59,431 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:59,431 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:38:59,441 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:59,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48406 deadline: 1731857999432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:59,442 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:59,442 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:59,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48390 deadline: 1731857999432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:59,442 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:59,442 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:38:59,442 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:59,442 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:59,442 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:38:59,447 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:59,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48474 deadline: 1731857999443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:59,447 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:59,447 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:59,447 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:38:59,500 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/A/e60042d919a04568be5b1728416beef3 2024-11-17T15:38:59,507 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/B/e7d510adc3e24559b6ab12953fc5ffa0 is 50, key is test_row_0/B:col10/1731857939090/Put/seqid=0 2024-11-17T15:38:59,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742280_1456 (size=12001) 2024-11-17T15:38:59,518 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/B/e7d510adc3e24559b6ab12953fc5ffa0 2024-11-17T15:38:59,525 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/C/528eef393c914d15a2fdd5474b1313f9 is 50, key is test_row_0/C:col10/1731857939090/Put/seqid=0 2024-11-17T15:38:59,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742281_1457 (size=12001) 2024-11-17T15:38:59,538 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/C/3e5a1aef52334141ac642133c0432a17 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/C/3e5a1aef52334141ac642133c0432a17 2024-11-17T15:38:59,542 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 420677de373b14e08d29085670121ff4/C of 420677de373b14e08d29085670121ff4 into 3e5a1aef52334141ac642133c0432a17(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T15:38:59,542 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 420677de373b14e08d29085670121ff4: 2024-11-17T15:38:59,542 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., storeName=420677de373b14e08d29085670121ff4/C, priority=13, startTime=1731857938691; duration=0sec 2024-11-17T15:38:59,542 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T15:38:59,542 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 420677de373b14e08d29085670121ff4:C 2024-11-17T15:38:59,554 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-17T15:38:59,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. 2024-11-17T15:38:59,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. as already flushing 2024-11-17T15:38:59,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. 2024-11-17T15:38:59,554 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:59,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:59,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:59,706 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-17T15:38:59,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. 2024-11-17T15:38:59,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. as already flushing 2024-11-17T15:38:59,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. 2024-11-17T15:38:59,707 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:59,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:59,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:59,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-17T15:38:59,750 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:59,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48458 deadline: 1731857999743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:59,750 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:59,750 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:59,750 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:38:59,759 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:59,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48406 deadline: 1731857999753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:59,760 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:59,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48390 deadline: 1731857999753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:59,760 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:59,760 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:59,760 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:38:59,760 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:38:59,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48474 deadline: 1731857999753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 2024-11-17T15:38:59,760 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:59,760 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:59,760 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:38:59,760 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:59,760 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:38:59,760 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:38:59,858 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-17T15:38:59,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. 2024-11-17T15:38:59,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. as already flushing 2024-11-17T15:38:59,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. 2024-11-17T15:38:59,859 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:59,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:59,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:38:59,930 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/C/528eef393c914d15a2fdd5474b1313f9 2024-11-17T15:38:59,934 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/A/e60042d919a04568be5b1728416beef3 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/A/e60042d919a04568be5b1728416beef3 2024-11-17T15:38:59,937 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/A/e60042d919a04568be5b1728416beef3, entries=150, sequenceid=76, filesize=11.7 K 2024-11-17T15:38:59,937 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/B/e7d510adc3e24559b6ab12953fc5ffa0 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/B/e7d510adc3e24559b6ab12953fc5ffa0 2024-11-17T15:38:59,940 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/B/e7d510adc3e24559b6ab12953fc5ffa0, entries=150, sequenceid=76, filesize=11.7 K 2024-11-17T15:38:59,941 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/C/528eef393c914d15a2fdd5474b1313f9 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/C/528eef393c914d15a2fdd5474b1313f9 2024-11-17T15:38:59,944 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/C/528eef393c914d15a2fdd5474b1313f9, entries=150, sequenceid=76, filesize=11.7 K 2024-11-17T15:38:59,945 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for 420677de373b14e08d29085670121ff4 in 855ms, sequenceid=76, compaction requested=false 2024-11-17T15:38:59,945 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 420677de373b14e08d29085670121ff4: 2024-11-17T15:39:00,011 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-17T15:39:00,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. 2024-11-17T15:39:00,011 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2902): Flushing 420677de373b14e08d29085670121ff4 3/3 column families, dataSize=40.25 KB heapSize=106.22 KB 2024-11-17T15:39:00,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/A/4805b565078b4a26bd2e96d309c9fdbe is 50, key is test_row_0/A:col10/1731857939103/Put/seqid=0 2024-11-17T15:39:00,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742282_1458 (size=12001) 2024-11-17T15:39:00,032 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=88 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/A/4805b565078b4a26bd2e96d309c9fdbe 2024-11-17T15:39:00,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/B/0b8d8baf03fb458bb96a3e4f636b4eee is 50, key is test_row_0/B:col10/1731857939103/Put/seqid=0 2024-11-17T15:39:00,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742283_1459 (size=12001) 2024-11-17T15:39:00,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-17T15:39:00,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(8855): Flush requested on 420677de373b14e08d29085670121ff4 2024-11-17T15:39:00,268 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. as already flushing 2024-11-17T15:39:00,327 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:00,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48390 deadline: 1731858000318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:00,327 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:00,327 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:00,327 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:39:00,331 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:00,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48474 deadline: 1731858000319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:00,332 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:00,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48458 deadline: 1731858000321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:00,332 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:00,332 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:00,332 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:39:00,332 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:00,332 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:00,332 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:00,332 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:39:00,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48406 deadline: 1731858000327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:00,332 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:00,332 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:00,333 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:39:00,441 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:00,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48390 deadline: 1731858000433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:00,442 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:00,442 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:00,442 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:39:00,452 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=88 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/B/0b8d8baf03fb458bb96a3e4f636b4eee 2024-11-17T15:39:00,453 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:00,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48474 deadline: 1731858000443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:00,453 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:00,453 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:00,453 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:00,453 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:39:00,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48458 deadline: 1731858000443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:00,453 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:00,453 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:00,454 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:39:00,453 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:00,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48406 deadline: 1731858000443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:00,454 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:00,454 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:00,454 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:39:00,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/C/1c77f3753b7c4b099451182a0f1eca2a is 50, key is test_row_0/C:col10/1731857939103/Put/seqid=0 2024-11-17T15:39:00,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742284_1460 (size=12001) 2024-11-17T15:39:00,651 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:00,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48390 deadline: 1731858000643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:00,651 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:00,651 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:00,651 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:39:00,672 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:00,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48474 deadline: 1731858000663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:00,673 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:00,673 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:00,673 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:00,673 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:39:00,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48458 deadline: 1731858000663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:00,673 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:00,673 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:00,673 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:00,673 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:39:00,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48406 deadline: 1731858000663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:00,674 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:00,674 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:00,674 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:39:00,863 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=88 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/C/1c77f3753b7c4b099451182a0f1eca2a 2024-11-17T15:39:00,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/A/4805b565078b4a26bd2e96d309c9fdbe as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/A/4805b565078b4a26bd2e96d309c9fdbe 2024-11-17T15:39:00,870 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/A/4805b565078b4a26bd2e96d309c9fdbe, entries=150, sequenceid=88, filesize=11.7 K 2024-11-17T15:39:00,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/B/0b8d8baf03fb458bb96a3e4f636b4eee as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/B/0b8d8baf03fb458bb96a3e4f636b4eee 2024-11-17T15:39:00,873 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/B/0b8d8baf03fb458bb96a3e4f636b4eee, entries=150, sequenceid=88, filesize=11.7 K 2024-11-17T15:39:00,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/C/1c77f3753b7c4b099451182a0f1eca2a as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/C/1c77f3753b7c4b099451182a0f1eca2a 2024-11-17T15:39:00,877 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/C/1c77f3753b7c4b099451182a0f1eca2a, entries=150, sequenceid=88, filesize=11.7 K 2024-11-17T15:39:00,878 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.17 KB/108720, currentSize=161.02 KB/164880 for 420677de373b14e08d29085670121ff4 in 867ms, sequenceid=88, compaction requested=true 2024-11-17T15:39:00,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2603): Flush status journal for 420677de373b14e08d29085670121ff4: 2024-11-17T15:39:00,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. 2024-11-17T15:39:00,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=126 2024-11-17T15:39:00,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.HMaster(4169): Remote procedure done, pid=126 2024-11-17T15:39:00,880 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-11-17T15:39:00,881 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7830 sec 2024-11-17T15:39:00,882 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees in 1.7870 sec 2024-11-17T15:39:00,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(8855): Flush requested on 420677de373b14e08d29085670121ff4 2024-11-17T15:39:00,957 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T15:39:00,958 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 420677de373b14e08d29085670121ff4 3/3 column families, dataSize=167.72 KB heapSize=440.20 KB 2024-11-17T15:39:00,963 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/A/4394999c3c434c6f8b2298469d5105ae is 50, key is test_row_0/A:col10/1731857940310/Put/seqid=0 2024-11-17T15:39:00,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742285_1461 (size=14341) 2024-11-17T15:39:00,989 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:00,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48474 deadline: 1731858000983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:00,990 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:00,990 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:00,990 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:00,990 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:39:00,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48458 deadline: 1731858000983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:00,990 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:00,990 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:00,990 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:39:00,990 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:00,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48406 deadline: 1731858000983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:00,990 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:00,990 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:00,990 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:39:00,990 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:00,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48390 deadline: 1731858000985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:00,991 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:00,991 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:00,991 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:39:01,100 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:01,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48390 deadline: 1731858001093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:01,101 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:01,101 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:01,101 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:39:01,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-17T15:39:01,243 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T15:39:01,244 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T15:39:01,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees 2024-11-17T15:39:01,245 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T15:39:01,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-17T15:39:01,246 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T15:39:01,246 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T15:39:01,321 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:01,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48390 deadline: 1731858001313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:01,321 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:01,321 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:01,321 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:39:01,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-17T15:39:01,366 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/A/4394999c3c434c6f8b2298469d5105ae 2024-11-17T15:39:01,373 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/B/a53430876a9a43e58b166401b120d17a is 50, key is test_row_0/B:col10/1731857940310/Put/seqid=0 2024-11-17T15:39:01,377 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742286_1462 (size=12001) 2024-11-17T15:39:01,398 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-17T15:39:01,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. 2024-11-17T15:39:01,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. as already flushing 2024-11-17T15:39:01,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. 2024-11-17T15:39:01,399 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:39:01,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:39:01,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:39:01,501 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:01,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48474 deadline: 1731858001493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:01,502 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:01,502 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:01,502 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:39:01,512 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:01,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48458 deadline: 1731858001503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:01,512 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:01,512 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:01,512 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:01,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48406 deadline: 1731858001503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:01,512 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:39:01,512 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:01,512 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:01,512 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:39:01,552 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-17T15:39:01,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. 2024-11-17T15:39:01,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. as already flushing 2024-11-17T15:39:01,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. 2024-11-17T15:39:01,552 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:39:01,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:39:01,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:39:01,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-17T15:39:01,629 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:01,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48390 deadline: 1731858001623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:01,630 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:01,630 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:01,630 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:39:01,705 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-17T15:39:01,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. 2024-11-17T15:39:01,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. as already flushing 2024-11-17T15:39:01,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. 2024-11-17T15:39:01,705 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:39:01,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:39:01,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:39:01,777 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/B/a53430876a9a43e58b166401b120d17a 2024-11-17T15:39:01,784 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/C/05801731e76845bd81dcf316e26027c3 is 50, key is test_row_0/C:col10/1731857940310/Put/seqid=0 2024-11-17T15:39:01,787 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742287_1463 (size=12001) 2024-11-17T15:39:01,858 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-17T15:39:01,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. 2024-11-17T15:39:01,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. as already flushing 2024-11-17T15:39:01,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. 2024-11-17T15:39:01,859 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:39:01,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:39:01,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:39:01,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-17T15:39:02,011 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-17T15:39:02,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. 2024-11-17T15:39:02,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. as already flushing 2024-11-17T15:39:02,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. 2024-11-17T15:39:02,011 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:39:02,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:39:02,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:39:02,141 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:02,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48390 deadline: 1731858002133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:02,141 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:02,141 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:02,141 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:39:02,164 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-17T15:39:02,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. 2024-11-17T15:39:02,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. as already flushing 2024-11-17T15:39:02,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. 2024-11-17T15:39:02,164 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:39:02,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:39:02,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:39:02,188 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/C/05801731e76845bd81dcf316e26027c3 2024-11-17T15:39:02,192 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/A/4394999c3c434c6f8b2298469d5105ae as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/A/4394999c3c434c6f8b2298469d5105ae 2024-11-17T15:39:02,195 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/A/4394999c3c434c6f8b2298469d5105ae, entries=200, sequenceid=116, filesize=14.0 K 2024-11-17T15:39:02,195 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/B/a53430876a9a43e58b166401b120d17a as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/B/a53430876a9a43e58b166401b120d17a 2024-11-17T15:39:02,198 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/B/a53430876a9a43e58b166401b120d17a, entries=150, sequenceid=116, filesize=11.7 K 2024-11-17T15:39:02,199 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/C/05801731e76845bd81dcf316e26027c3 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/C/05801731e76845bd81dcf316e26027c3 2024-11-17T15:39:02,202 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/C/05801731e76845bd81dcf316e26027c3, entries=150, sequenceid=116, filesize=11.7 K 2024-11-17T15:39:02,203 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=33.54 KB/34350 for 420677de373b14e08d29085670121ff4 in 1245ms, sequenceid=116, compaction requested=true 2024-11-17T15:39:02,203 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 420677de373b14e08d29085670121ff4: 2024-11-17T15:39:02,203 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 420677de373b14e08d29085670121ff4:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T15:39:02,203 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T15:39:02,203 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 420677de373b14e08d29085670121ff4:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T15:39:02,203 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T15:39:02,203 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 420677de373b14e08d29085670121ff4:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T15:39:02,203 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T15:39:02,203 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T15:39:02,203 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T15:39:02,204 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50447 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T15:39:02,204 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1541): 420677de373b14e08d29085670121ff4/A is initiating minor compaction (all files) 2024-11-17T15:39:02,204 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 420677de373b14e08d29085670121ff4/A in TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. 2024-11-17T15:39:02,204 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/A/bb8bbbc8baae426890539d5d7bed9c5f, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/A/e60042d919a04568be5b1728416beef3, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/A/4805b565078b4a26bd2e96d309c9fdbe, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/A/4394999c3c434c6f8b2298469d5105ae] into tmpdir=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp, totalSize=49.3 K 2024-11-17T15:39:02,204 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting bb8bbbc8baae426890539d5d7bed9c5f, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731857937199 2024-11-17T15:39:02,205 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting e60042d919a04568be5b1728416beef3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731857937917 2024-11-17T15:39:02,205 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T15:39:02,205 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4805b565078b4a26bd2e96d309c9fdbe, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1731857939091 2024-11-17T15:39:02,205 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1541): 420677de373b14e08d29085670121ff4/B is initiating minor compaction (all files) 2024-11-17T15:39:02,205 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 420677de373b14e08d29085670121ff4/B in TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. 2024-11-17T15:39:02,205 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/B/83e66da322354814add77a9da6c7c038, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/B/e7d510adc3e24559b6ab12953fc5ffa0, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/B/0b8d8baf03fb458bb96a3e4f636b4eee, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/B/a53430876a9a43e58b166401b120d17a] into tmpdir=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp, totalSize=47.0 K 2024-11-17T15:39:02,205 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4394999c3c434c6f8b2298469d5105ae, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731857940310 2024-11-17T15:39:02,205 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting 83e66da322354814add77a9da6c7c038, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731857937199 2024-11-17T15:39:02,206 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting e7d510adc3e24559b6ab12953fc5ffa0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731857937917 2024-11-17T15:39:02,206 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting 0b8d8baf03fb458bb96a3e4f636b4eee, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1731857939091 2024-11-17T15:39:02,206 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting a53430876a9a43e58b166401b120d17a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731857940310 2024-11-17T15:39:02,213 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 420677de373b14e08d29085670121ff4#A#compaction#386 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T15:39:02,214 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/A/10f43bae05674d589709bd59a71522b0 is 50, key is test_row_0/A:col10/1731857940310/Put/seqid=0 2024-11-17T15:39:02,217 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 420677de373b14e08d29085670121ff4#B#compaction#387 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T15:39:02,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742288_1464 (size=12241) 2024-11-17T15:39:02,218 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/B/69c77f3423dc4110965c7be2515679aa is 50, key is test_row_0/B:col10/1731857940310/Put/seqid=0 2024-11-17T15:39:02,225 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/A/10f43bae05674d589709bd59a71522b0 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/A/10f43bae05674d589709bd59a71522b0 2024-11-17T15:39:02,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742289_1465 (size=12241) 2024-11-17T15:39:02,233 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/B/69c77f3423dc4110965c7be2515679aa as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/B/69c77f3423dc4110965c7be2515679aa 2024-11-17T15:39:02,234 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 420677de373b14e08d29085670121ff4/A of 420677de373b14e08d29085670121ff4 into 10f43bae05674d589709bd59a71522b0(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T15:39:02,234 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 420677de373b14e08d29085670121ff4: 2024-11-17T15:39:02,234 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., storeName=420677de373b14e08d29085670121ff4/A, priority=12, startTime=1731857942203; duration=0sec 2024-11-17T15:39:02,234 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T15:39:02,234 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 420677de373b14e08d29085670121ff4:A 2024-11-17T15:39:02,234 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T15:39:02,235 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T15:39:02,235 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1541): 420677de373b14e08d29085670121ff4/C is initiating minor compaction (all files) 2024-11-17T15:39:02,235 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 420677de373b14e08d29085670121ff4/C in TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. 2024-11-17T15:39:02,235 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/C/3e5a1aef52334141ac642133c0432a17, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/C/528eef393c914d15a2fdd5474b1313f9, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/C/1c77f3753b7c4b099451182a0f1eca2a, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/C/05801731e76845bd81dcf316e26027c3] into tmpdir=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp, totalSize=47.0 K 2024-11-17T15:39:02,236 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3e5a1aef52334141ac642133c0432a17, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731857937199 2024-11-17T15:39:02,239 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 528eef393c914d15a2fdd5474b1313f9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731857937917 2024-11-17T15:39:02,240 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1c77f3753b7c4b099451182a0f1eca2a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1731857939091 2024-11-17T15:39:02,241 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 05801731e76845bd81dcf316e26027c3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731857940310 2024-11-17T15:39:02,241 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 420677de373b14e08d29085670121ff4/B of 420677de373b14e08d29085670121ff4 into 69c77f3423dc4110965c7be2515679aa(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T15:39:02,241 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 420677de373b14e08d29085670121ff4: 2024-11-17T15:39:02,241 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., storeName=420677de373b14e08d29085670121ff4/B, priority=12, startTime=1731857942203; duration=0sec 2024-11-17T15:39:02,241 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T15:39:02,241 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 420677de373b14e08d29085670121ff4:B 2024-11-17T15:39:02,250 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 420677de373b14e08d29085670121ff4#C#compaction#388 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T15:39:02,251 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/C/fea0a1f0733448d98beb225b2135f127 is 50, key is test_row_0/C:col10/1731857940310/Put/seqid=0 2024-11-17T15:39:02,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742290_1466 (size=12241) 2024-11-17T15:39:02,265 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/C/fea0a1f0733448d98beb225b2135f127 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/C/fea0a1f0733448d98beb225b2135f127 2024-11-17T15:39:02,271 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 420677de373b14e08d29085670121ff4/C of 420677de373b14e08d29085670121ff4 into fea0a1f0733448d98beb225b2135f127(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T15:39:02,271 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 420677de373b14e08d29085670121ff4: 2024-11-17T15:39:02,271 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., storeName=420677de373b14e08d29085670121ff4/C, priority=12, startTime=1731857942203; duration=0sec 2024-11-17T15:39:02,271 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T15:39:02,271 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 420677de373b14e08d29085670121ff4:C 2024-11-17T15:39:02,317 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-17T15:39:02,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. 2024-11-17T15:39:02,317 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2902): Flushing 420677de373b14e08d29085670121ff4 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-17T15:39:02,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/A/09d43038f697433782aaa7db76a91ea7 is 50, key is test_row_0/A:col10/1731857940968/Put/seqid=0 2024-11-17T15:39:02,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742291_1467 (size=12001) 2024-11-17T15:39:02,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-17T15:39:02,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(8855): Flush requested on 420677de373b14e08d29085670121ff4 2024-11-17T15:39:02,526 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. as already flushing 2024-11-17T15:39:02,595 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:02,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48406 deadline: 1731858002584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:02,595 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:02,595 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:02,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48474 deadline: 1731858002586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:02,595 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:02,595 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:39:02,595 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:02,595 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:02,596 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:39:02,596 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:02,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48458 deadline: 1731858002588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:02,597 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:02,597 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:02,597 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:39:02,706 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:02,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48406 deadline: 1731858002703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:02,706 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:02,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48474 deadline: 1731858002703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:02,706 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:02,706 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:02,706 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:39:02,706 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:02,706 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:02,706 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:02,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48458 deadline: 1731858002703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:02,706 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:39:02,707 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:02,707 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:02,707 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:39:02,735 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=127 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/A/09d43038f697433782aaa7db76a91ea7 2024-11-17T15:39:02,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/B/4abb95086c6d46b2962c6a111ee546ad is 50, key is test_row_0/B:col10/1731857940968/Put/seqid=0 2024-11-17T15:39:02,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742292_1468 (size=12001) 2024-11-17T15:39:02,921 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:02,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48406 deadline: 1731858002913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:02,921 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:02,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48474 deadline: 1731858002913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:02,921 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:02,921 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:02,921 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:39:02,921 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:02,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48458 deadline: 1731858002913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:02,922 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:02,922 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:02,922 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:39:02,922 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:02,922 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:02,922 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:39:03,146 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=127 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/B/4abb95086c6d46b2962c6a111ee546ad 2024-11-17T15:39:03,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/C/fc36bd9960e0461ba3b57b8389955576 is 50, key is test_row_0/C:col10/1731857940968/Put/seqid=0 2024-11-17T15:39:03,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742293_1469 (size=12001) 2024-11-17T15:39:03,161 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:03,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48390 deadline: 1731858003153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:03,161 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 7a780d55532c,32955,1731857800648 for 'test_row_2' in 420677de373b14e08d29085670121ff4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2176 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T15:39:03,162 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:03,162 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:03,162 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:39:03,239 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:03,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48406 deadline: 1731858003233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:03,239 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:03,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48474 deadline: 1731858003233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:03,239 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:03,239 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:03,239 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:39:03,240 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:03,240 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:03,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48458 deadline: 1731858003233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:03,240 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:03,240 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:39:03,240 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:03,240 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:03,240 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:39:03,260 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:03,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48442 deadline: 1731858003253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:03,260 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 7a780d55532c,32955,1731857800648 for 'test_row_0' in 420677de373b14e08d29085670121ff4 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8234 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T15:39:03,261 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:03,261 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:03,261 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:39:03,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-17T15:39:03,556 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=127 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/C/fc36bd9960e0461ba3b57b8389955576 2024-11-17T15:39:03,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/A/09d43038f697433782aaa7db76a91ea7 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/A/09d43038f697433782aaa7db76a91ea7 2024-11-17T15:39:03,564 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/A/09d43038f697433782aaa7db76a91ea7, entries=150, sequenceid=127, filesize=11.7 K 2024-11-17T15:39:03,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/B/4abb95086c6d46b2962c6a111ee546ad as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/B/4abb95086c6d46b2962c6a111ee546ad 2024-11-17T15:39:03,567 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/B/4abb95086c6d46b2962c6a111ee546ad, entries=150, sequenceid=127, filesize=11.7 K 2024-11-17T15:39:03,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/C/fc36bd9960e0461ba3b57b8389955576 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/C/fc36bd9960e0461ba3b57b8389955576 2024-11-17T15:39:03,571 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/C/fc36bd9960e0461ba3b57b8389955576, entries=150, sequenceid=127, filesize=11.7 K 2024-11-17T15:39:03,572 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=167.72 KB/171750 for 420677de373b14e08d29085670121ff4 in 1255ms, sequenceid=127, compaction requested=false 2024-11-17T15:39:03,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2603): Flush status journal for 420677de373b14e08d29085670121ff4: 2024-11-17T15:39:03,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. 2024-11-17T15:39:03,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-11-17T15:39:03,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.HMaster(4169): Remote procedure done, pid=128 2024-11-17T15:39:03,575 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-11-17T15:39:03,575 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3280 sec 2024-11-17T15:39:03,577 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees in 2.3320 sec 2024-11-17T15:39:03,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(8855): Flush requested on 420677de373b14e08d29085670121ff4 2024-11-17T15:39:03,759 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T15:39:03,759 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 420677de373b14e08d29085670121ff4 3/3 column families, dataSize=181.14 KB heapSize=475.36 KB 2024-11-17T15:39:03,765 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/A/867380b104334bdcaf817efa0fd6eb23 is 50, key is test_row_0/A:col10/1731857943758/Put/seqid=0 2024-11-17T15:39:03,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742294_1470 (size=19321) 2024-11-17T15:39:03,774 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:03,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48474 deadline: 1731858003770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:03,774 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:03,775 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:03,775 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:39:03,779 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:03,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48458 deadline: 1731858003771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:03,779 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:03,779 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:03,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48406 deadline: 1731858003771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:03,780 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:03,780 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:39:03,781 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:03,781 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:03,781 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:39:03,887 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:03,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48458 deadline: 1731858003883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:03,887 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:03,888 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:03,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48474 deadline: 1731858003883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:03,888 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:03,888 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:39:03,888 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:03,888 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:03,888 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:03,888 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:39:03,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48406 deadline: 1731858003883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:03,888 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:03,888 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:03,888 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:39:04,097 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:04,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48458 deadline: 1731858004093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:04,098 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:04,098 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:04,098 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:04,098 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:39:04,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48474 deadline: 1731858004093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:04,098 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:04,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48406 deadline: 1731858004093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:04,098 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:04,098 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:04,098 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:39:04,099 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:04,099 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:04,099 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:39:04,170 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=62.62 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/A/867380b104334bdcaf817efa0fd6eb23 2024-11-17T15:39:04,177 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/B/2b1e1c82825e4af08678b9857d72a0d2 is 50, key is test_row_0/B:col10/1731857943758/Put/seqid=0 2024-11-17T15:39:04,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742295_1471 (size=12151) 2024-11-17T15:39:04,181 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=62.62 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/B/2b1e1c82825e4af08678b9857d72a0d2 2024-11-17T15:39:04,188 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/C/983dee40eac14c3e99314204d759d9ee is 50, key is test_row_0/C:col10/1731857943758/Put/seqid=0 2024-11-17T15:39:04,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742296_1472 (size=12151) 2024-11-17T15:39:04,194 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=62.62 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/C/983dee40eac14c3e99314204d759d9ee 2024-11-17T15:39:04,197 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/A/867380b104334bdcaf817efa0fd6eb23 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/A/867380b104334bdcaf817efa0fd6eb23 2024-11-17T15:39:04,200 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/A/867380b104334bdcaf817efa0fd6eb23, entries=300, sequenceid=158, filesize=18.9 K 2024-11-17T15:39:04,201 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/B/2b1e1c82825e4af08678b9857d72a0d2 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/B/2b1e1c82825e4af08678b9857d72a0d2 2024-11-17T15:39:04,204 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/B/2b1e1c82825e4af08678b9857d72a0d2, entries=150, sequenceid=158, filesize=11.9 K 2024-11-17T15:39:04,205 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/C/983dee40eac14c3e99314204d759d9ee as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/C/983dee40eac14c3e99314204d759d9ee 2024-11-17T15:39:04,208 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/C/983dee40eac14c3e99314204d759d9ee, entries=150, sequenceid=158, filesize=11.9 K 2024-11-17T15:39:04,209 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~187.85 KB/192360, heapSize ~492.89 KB/504720, currentSize=20.13 KB/20610 for 420677de373b14e08d29085670121ff4 in 449ms, sequenceid=158, compaction requested=true 2024-11-17T15:39:04,209 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 420677de373b14e08d29085670121ff4: 2024-11-17T15:39:04,209 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 420677de373b14e08d29085670121ff4:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T15:39:04,209 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T15:39:04,209 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T15:39:04,209 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T15:39:04,209 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 420677de373b14e08d29085670121ff4:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T15:39:04,209 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T15:39:04,209 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 420677de373b14e08d29085670121ff4:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T15:39:04,209 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T15:39:04,210 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 43563 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T15:39:04,210 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36393 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T15:39:04,210 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1541): 420677de373b14e08d29085670121ff4/A is initiating minor compaction (all files) 2024-11-17T15:39:04,210 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1541): 420677de373b14e08d29085670121ff4/B is initiating minor compaction (all files) 2024-11-17T15:39:04,210 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 420677de373b14e08d29085670121ff4/A in TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. 2024-11-17T15:39:04,210 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 420677de373b14e08d29085670121ff4/B in TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. 2024-11-17T15:39:04,210 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/A/10f43bae05674d589709bd59a71522b0, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/A/09d43038f697433782aaa7db76a91ea7, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/A/867380b104334bdcaf817efa0fd6eb23] into tmpdir=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp, totalSize=42.5 K 2024-11-17T15:39:04,210 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/B/69c77f3423dc4110965c7be2515679aa, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/B/4abb95086c6d46b2962c6a111ee546ad, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/B/2b1e1c82825e4af08678b9857d72a0d2] into tmpdir=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp, totalSize=35.5 K 2024-11-17T15:39:04,210 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 10f43bae05674d589709bd59a71522b0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731857940310 2024-11-17T15:39:04,210 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting 69c77f3423dc4110965c7be2515679aa, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731857940310 2024-11-17T15:39:04,210 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 09d43038f697433782aaa7db76a91ea7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1731857940968 2024-11-17T15:39:04,210 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting 4abb95086c6d46b2962c6a111ee546ad, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1731857940968 2024-11-17T15:39:04,211 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 867380b104334bdcaf817efa0fd6eb23, keycount=300, bloomtype=ROW, size=18.9 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1731857942582 2024-11-17T15:39:04,211 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting 2b1e1c82825e4af08678b9857d72a0d2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1731857942586 2024-11-17T15:39:04,222 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 420677de373b14e08d29085670121ff4#A#compaction#396 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T15:39:04,222 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 420677de373b14e08d29085670121ff4#B#compaction#395 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T15:39:04,222 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/A/31f2e33148a74c68ad38cdfa77f5dd2a is 50, key is test_row_0/A:col10/1731857943758/Put/seqid=0 2024-11-17T15:39:04,222 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/B/7190ae69d3554889aa12e1235f491bb8 is 50, key is test_row_0/B:col10/1731857943758/Put/seqid=0 2024-11-17T15:39:04,227 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742297_1473 (size=12493) 2024-11-17T15:39:04,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742298_1474 (size=12493) 2024-11-17T15:39:04,234 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/B/7190ae69d3554889aa12e1235f491bb8 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/B/7190ae69d3554889aa12e1235f491bb8 2024-11-17T15:39:04,238 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 420677de373b14e08d29085670121ff4/B of 420677de373b14e08d29085670121ff4 into 7190ae69d3554889aa12e1235f491bb8(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T15:39:04,238 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 420677de373b14e08d29085670121ff4: 2024-11-17T15:39:04,238 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., storeName=420677de373b14e08d29085670121ff4/B, priority=13, startTime=1731857944209; duration=0sec 2024-11-17T15:39:04,238 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T15:39:04,238 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 420677de373b14e08d29085670121ff4:B 2024-11-17T15:39:04,238 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T15:39:04,239 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36393 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T15:39:04,239 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1541): 420677de373b14e08d29085670121ff4/C is initiating minor compaction (all files) 2024-11-17T15:39:04,239 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 420677de373b14e08d29085670121ff4/C in TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. 2024-11-17T15:39:04,239 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/C/fea0a1f0733448d98beb225b2135f127, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/C/fc36bd9960e0461ba3b57b8389955576, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/C/983dee40eac14c3e99314204d759d9ee] into tmpdir=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp, totalSize=35.5 K 2024-11-17T15:39:04,240 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting fea0a1f0733448d98beb225b2135f127, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731857940310 2024-11-17T15:39:04,240 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting fc36bd9960e0461ba3b57b8389955576, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1731857940968 2024-11-17T15:39:04,240 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting 983dee40eac14c3e99314204d759d9ee, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1731857942586 2024-11-17T15:39:04,245 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 420677de373b14e08d29085670121ff4#C#compaction#397 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T15:39:04,246 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/C/7ee62f6a879e45a182ca1171bfaec364 is 50, key is test_row_0/C:col10/1731857943758/Put/seqid=0 2024-11-17T15:39:04,251 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742299_1475 (size=12493) 2024-11-17T15:39:04,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(8855): Flush requested on 420677de373b14e08d29085670121ff4 2024-11-17T15:39:04,414 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T15:39:04,414 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 420677de373b14e08d29085670121ff4 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-17T15:39:04,419 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/A/5fbd9a41a7914715a6a3a47d2f58389c is 50, key is test_row_0/A:col10/1731857944412/Put/seqid=0 2024-11-17T15:39:04,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742300_1476 (size=12151) 2024-11-17T15:39:04,492 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:04,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48458 deadline: 1731858004479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:04,493 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:04,493 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:04,493 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:39:04,495 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:04,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48474 deadline: 1731858004482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:04,496 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:04,496 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:04,496 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:39:04,502 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:04,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48406 deadline: 1731858004492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:04,503 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:04,503 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:04,503 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:39:04,611 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:04,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48458 deadline: 1731858004603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:04,611 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:04,611 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:04,611 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:04,611 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:39:04,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48474 deadline: 1731858004603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:04,612 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:04,612 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:04,612 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:39:04,624 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:04,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48406 deadline: 1731858004613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:04,625 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:04,625 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:04,625 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:39:04,639 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/A/31f2e33148a74c68ad38cdfa77f5dd2a as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/A/31f2e33148a74c68ad38cdfa77f5dd2a 2024-11-17T15:39:04,643 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 420677de373b14e08d29085670121ff4/A of 420677de373b14e08d29085670121ff4 into 31f2e33148a74c68ad38cdfa77f5dd2a(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T15:39:04,643 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 420677de373b14e08d29085670121ff4: 2024-11-17T15:39:04,643 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., storeName=420677de373b14e08d29085670121ff4/A, priority=13, startTime=1731857944209; duration=0sec 2024-11-17T15:39:04,643 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T15:39:04,643 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 420677de373b14e08d29085670121ff4:A 2024-11-17T15:39:04,655 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/C/7ee62f6a879e45a182ca1171bfaec364 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/C/7ee62f6a879e45a182ca1171bfaec364 2024-11-17T15:39:04,659 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 420677de373b14e08d29085670121ff4/C of 420677de373b14e08d29085670121ff4 into 7ee62f6a879e45a182ca1171bfaec364(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T15:39:04,659 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 420677de373b14e08d29085670121ff4: 2024-11-17T15:39:04,659 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., storeName=420677de373b14e08d29085670121ff4/C, priority=13, startTime=1731857944209; duration=0sec 2024-11-17T15:39:04,659 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T15:39:04,659 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 420677de373b14e08d29085670121ff4:C 2024-11-17T15:39:04,828 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/A/5fbd9a41a7914715a6a3a47d2f58389c 2024-11-17T15:39:04,828 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:04,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48458 deadline: 1731858004823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:04,829 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:04,829 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:04,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48474 deadline: 1731858004823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:04,829 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:04,829 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:39:04,829 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:04,829 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:04,829 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:39:04,834 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/B/7b8d6451dfa249f8b7f7c31ef8c9ba64 is 50, key is test_row_0/B:col10/1731857944412/Put/seqid=0 2024-11-17T15:39:04,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742301_1477 (size=12151) 2024-11-17T15:39:04,840 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:04,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48406 deadline: 1731858004833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:04,840 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:04,840 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:04,840 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:39:05,138 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:05,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48458 deadline: 1731858005133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:05,139 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:05,139 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:05,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48474 deadline: 1731858005133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:05,139 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:05,139 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:39:05,139 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:05,139 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:05,139 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:39:05,149 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:05,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48406 deadline: 1731858005142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:05,150 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:05,150 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:05,150 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:39:05,181 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:05,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48390 deadline: 1731858005173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:05,181 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 7a780d55532c,32955,1731857800648 for 'test_row_2' in 420677de373b14e08d29085670121ff4 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4196 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T15:39:05,182 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:05,182 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:05,182 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:39:05,239 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/B/7b8d6451dfa249f8b7f7c31ef8c9ba64 2024-11-17T15:39:05,246 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/C/c2b293984fc445828812b5b511b8a53a is 50, key is test_row_0/C:col10/1731857944412/Put/seqid=0 2024-11-17T15:39:05,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742302_1478 (size=12151) 2024-11-17T15:39:05,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-17T15:39:05,413 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T15:39:05,413 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T15:39:05,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees 2024-11-17T15:39:05,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-17T15:39:05,415 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T15:39:05,415 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T15:39:05,416 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T15:39:05,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-17T15:39:05,567 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-17T15:39:05,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. 2024-11-17T15:39:05,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. as already flushing 2024-11-17T15:39:05,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. 2024-11-17T15:39:05,568 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:39:05,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:39:05,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:39:05,650 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:05,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48458 deadline: 1731858005643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:05,650 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:05,650 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:05,650 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:39:05,653 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/C/c2b293984fc445828812b5b511b8a53a 2024-11-17T15:39:05,657 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/A/5fbd9a41a7914715a6a3a47d2f58389c as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/A/5fbd9a41a7914715a6a3a47d2f58389c 2024-11-17T15:39:05,660 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:05,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48474 deadline: 1731858005653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:05,661 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:05,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48406 deadline: 1731858005653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:05,661 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:05,661 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:05,661 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:39:05,661 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:05,661 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:05,661 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:39:05,662 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/A/5fbd9a41a7914715a6a3a47d2f58389c, entries=150, sequenceid=170, filesize=11.9 K 2024-11-17T15:39:05,663 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/B/7b8d6451dfa249f8b7f7c31ef8c9ba64 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/B/7b8d6451dfa249f8b7f7c31ef8c9ba64 2024-11-17T15:39:05,667 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/B/7b8d6451dfa249f8b7f7c31ef8c9ba64, entries=150, sequenceid=170, filesize=11.9 K 2024-11-17T15:39:05,667 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/C/c2b293984fc445828812b5b511b8a53a as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/C/c2b293984fc445828812b5b511b8a53a 2024-11-17T15:39:05,671 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/C/c2b293984fc445828812b5b511b8a53a, entries=150, sequenceid=170, filesize=11.9 K 2024-11-17T15:39:05,672 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 420677de373b14e08d29085670121ff4 in 1258ms, sequenceid=170, compaction requested=false 2024-11-17T15:39:05,672 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 420677de373b14e08d29085670121ff4: 2024-11-17T15:39:05,720 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-17T15:39:05,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. 2024-11-17T15:39:05,721 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2902): Flushing 420677de373b14e08d29085670121ff4 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-17T15:39:05,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/A/8827c182f10443518368b2bf73442781 is 50, key is test_row_0/A:col10/1731857944476/Put/seqid=0 2024-11-17T15:39:05,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742303_1479 (size=12151) 2024-11-17T15:39:05,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-17T15:39:06,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-17T15:39:06,129 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/A/8827c182f10443518368b2bf73442781 2024-11-17T15:39:06,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/B/c8f4cde97c46447b84e2c92845b60c4a is 50, key is test_row_0/B:col10/1731857944476/Put/seqid=0 2024-11-17T15:39:06,140 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742304_1480 (size=12151) 2024-11-17T15:39:06,541 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/B/c8f4cde97c46447b84e2c92845b60c4a 2024-11-17T15:39:06,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/C/bcc5ee4ef2cd489482d25b0178860951 is 50, key is test_row_0/C:col10/1731857944476/Put/seqid=0 2024-11-17T15:39:06,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742305_1481 (size=12151) 2024-11-17T15:39:06,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-17T15:39:06,553 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/C/bcc5ee4ef2cd489482d25b0178860951 2024-11-17T15:39:06,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/A/8827c182f10443518368b2bf73442781 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/A/8827c182f10443518368b2bf73442781 2024-11-17T15:39:06,561 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/A/8827c182f10443518368b2bf73442781, entries=150, sequenceid=197, filesize=11.9 K 2024-11-17T15:39:06,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/B/c8f4cde97c46447b84e2c92845b60c4a as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/B/c8f4cde97c46447b84e2c92845b60c4a 2024-11-17T15:39:06,565 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/B/c8f4cde97c46447b84e2c92845b60c4a, entries=150, sequenceid=197, filesize=11.9 K 2024-11-17T15:39:06,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/C/bcc5ee4ef2cd489482d25b0178860951 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/C/bcc5ee4ef2cd489482d25b0178860951 2024-11-17T15:39:06,569 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/C/bcc5ee4ef2cd489482d25b0178860951, entries=150, sequenceid=197, filesize=11.9 K 2024-11-17T15:39:06,569 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=0 B/0 for 420677de373b14e08d29085670121ff4 in 848ms, sequenceid=197, compaction requested=true 2024-11-17T15:39:06,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2603): Flush status journal for 420677de373b14e08d29085670121ff4: 2024-11-17T15:39:06,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. 2024-11-17T15:39:06,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=130 2024-11-17T15:39:06,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.HMaster(4169): Remote procedure done, pid=130 2024-11-17T15:39:06,572 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-11-17T15:39:06,573 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1550 sec 2024-11-17T15:39:06,574 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees in 1.1600 sec 2024-11-17T15:39:06,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(8855): Flush requested on 420677de373b14e08d29085670121ff4 2024-11-17T15:39:06,673 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T15:39:06,674 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 420677de373b14e08d29085670121ff4 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-17T15:39:06,678 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/A/52fca7712eaf44f280c61ddd400d6e96 is 50, key is test_row_0/A:col10/1731857946671/Put/seqid=0 2024-11-17T15:39:06,683 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742306_1482 (size=16931) 2024-11-17T15:39:06,753 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:06,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48406 deadline: 1731858006741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:06,753 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:06,753 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:06,753 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:06,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48458 deadline: 1731858006742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:06,753 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:39:06,753 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:06,753 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:06,753 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:39:06,758 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:06,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48474 deadline: 1731858006745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:06,758 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:06,758 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:06,758 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:39:06,873 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:06,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48406 deadline: 1731858006863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:06,873 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:06,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48458 deadline: 1731858006863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:06,873 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:06,873 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:06,873 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:39:06,873 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:06,873 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:06,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48474 deadline: 1731858006863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:06,873 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:06,873 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:39:06,874 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:06,874 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:06,874 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:39:07,084 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/A/52fca7712eaf44f280c61ddd400d6e96 2024-11-17T15:39:07,090 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/B/dd9f96e826f84a7d8486fa982a959410 is 50, key is test_row_0/B:col10/1731857946671/Put/seqid=0 2024-11-17T15:39:07,092 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:07,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48406 deadline: 1731858007082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:07,092 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:07,092 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:07,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48458 deadline: 1731858007083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:07,092 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:07,092 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:39:07,093 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:07,093 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:07,093 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:07,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48474 deadline: 1731858007083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:07,093 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:39:07,093 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:07,093 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:07,093 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:39:07,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742307_1483 (size=12151) 2024-11-17T15:39:07,411 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:07,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48406 deadline: 1731858007403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:07,411 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:07,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48458 deadline: 1731858007403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:07,412 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:07,412 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:07,412 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:39:07,412 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:07,412 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:07,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48474 deadline: 1731858007403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:07,412 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:07,412 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:39:07,412 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:07,412 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:07,412 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:39:07,494 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/B/dd9f96e826f84a7d8486fa982a959410 2024-11-17T15:39:07,500 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/C/26677ca8f30549be9190fce2c95fbd59 is 50, key is test_row_0/C:col10/1731857946671/Put/seqid=0 2024-11-17T15:39:07,504 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742308_1484 (size=12151) 2024-11-17T15:39:07,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-17T15:39:07,563 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T15:39:07,563 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T15:39:07,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees 2024-11-17T15:39:07,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-17T15:39:07,565 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T15:39:07,566 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T15:39:07,566 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T15:39:07,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-17T15:39:07,718 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-17T15:39:07,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. 2024-11-17T15:39:07,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. as already flushing 2024-11-17T15:39:07,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. 2024-11-17T15:39:07,718 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:39:07,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:39:07,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:39:07,870 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-17T15:39:07,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. 2024-11-17T15:39:07,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. as already flushing 2024-11-17T15:39:07,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. 2024-11-17T15:39:07,871 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:39:07,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:39:07,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:39:07,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-17T15:39:07,904 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/C/26677ca8f30549be9190fce2c95fbd59 2024-11-17T15:39:07,908 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/A/52fca7712eaf44f280c61ddd400d6e96 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/A/52fca7712eaf44f280c61ddd400d6e96 2024-11-17T15:39:07,911 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/A/52fca7712eaf44f280c61ddd400d6e96, entries=250, sequenceid=208, filesize=16.5 K 2024-11-17T15:39:07,912 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/B/dd9f96e826f84a7d8486fa982a959410 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/B/dd9f96e826f84a7d8486fa982a959410 2024-11-17T15:39:07,915 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/B/dd9f96e826f84a7d8486fa982a959410, entries=150, sequenceid=208, filesize=11.9 K 2024-11-17T15:39:07,916 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/C/26677ca8f30549be9190fce2c95fbd59 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/C/26677ca8f30549be9190fce2c95fbd59 2024-11-17T15:39:07,918 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/C/26677ca8f30549be9190fce2c95fbd59, entries=150, sequenceid=208, filesize=11.9 K 2024-11-17T15:39:07,919 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 420677de373b14e08d29085670121ff4 in 1246ms, sequenceid=208, compaction requested=true 2024-11-17T15:39:07,919 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 420677de373b14e08d29085670121ff4: 2024-11-17T15:39:07,919 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 420677de373b14e08d29085670121ff4:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T15:39:07,919 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T15:39:07,919 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 420677de373b14e08d29085670121ff4:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T15:39:07,919 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T15:39:07,919 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 420677de373b14e08d29085670121ff4:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T15:39:07,919 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T15:39:07,919 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T15:39:07,919 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T15:39:07,920 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T15:39:07,920 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53726 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T15:39:07,920 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1541): 420677de373b14e08d29085670121ff4/A is initiating minor compaction (all files) 2024-11-17T15:39:07,920 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1541): 420677de373b14e08d29085670121ff4/B is initiating minor compaction (all files) 2024-11-17T15:39:07,920 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 420677de373b14e08d29085670121ff4/A in TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. 2024-11-17T15:39:07,920 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 420677de373b14e08d29085670121ff4/B in TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. 2024-11-17T15:39:07,920 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/A/31f2e33148a74c68ad38cdfa77f5dd2a, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/A/5fbd9a41a7914715a6a3a47d2f58389c, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/A/8827c182f10443518368b2bf73442781, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/A/52fca7712eaf44f280c61ddd400d6e96] into tmpdir=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp, totalSize=52.5 K 2024-11-17T15:39:07,920 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/B/7190ae69d3554889aa12e1235f491bb8, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/B/7b8d6451dfa249f8b7f7c31ef8c9ba64, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/B/c8f4cde97c46447b84e2c92845b60c4a, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/B/dd9f96e826f84a7d8486fa982a959410] into tmpdir=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp, totalSize=47.8 K 2024-11-17T15:39:07,921 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting 7190ae69d3554889aa12e1235f491bb8, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1731857942586 2024-11-17T15:39:07,921 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 31f2e33148a74c68ad38cdfa77f5dd2a, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1731857942586 2024-11-17T15:39:07,921 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting 7b8d6451dfa249f8b7f7c31ef8c9ba64, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731857944406 2024-11-17T15:39:07,921 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5fbd9a41a7914715a6a3a47d2f58389c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731857944406 2024-11-17T15:39:07,921 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting c8f4cde97c46447b84e2c92845b60c4a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1731857944476 2024-11-17T15:39:07,921 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8827c182f10443518368b2bf73442781, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1731857944476 2024-11-17T15:39:07,921 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting dd9f96e826f84a7d8486fa982a959410, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1731857946664 2024-11-17T15:39:07,921 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 52fca7712eaf44f280c61ddd400d6e96, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1731857946662 2024-11-17T15:39:07,929 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 420677de373b14e08d29085670121ff4#A#compaction#407 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T15:39:07,929 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 420677de373b14e08d29085670121ff4#B#compaction#408 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T15:39:07,930 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/A/0721721744834162a605031969826d25 is 50, key is test_row_0/A:col10/1731857946671/Put/seqid=0 2024-11-17T15:39:07,930 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/B/1e3287eebbf24045a49d143486988d98 is 50, key is test_row_0/B:col10/1731857946671/Put/seqid=0 2024-11-17T15:39:07,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(8855): Flush requested on 420677de373b14e08d29085670121ff4 2024-11-17T15:39:07,932 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T15:39:07,932 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 420677de373b14e08d29085670121ff4 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-17T15:39:07,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742309_1485 (size=12629) 2024-11-17T15:39:07,938 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/A/bd253feab9074b2cabea2a742df66b24 is 50, key is test_row_0/A:col10/1731857946739/Put/seqid=0 2024-11-17T15:39:07,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742310_1486 (size=12629) 2024-11-17T15:39:07,944 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742311_1487 (size=12151) 2024-11-17T15:39:07,944 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/A/0721721744834162a605031969826d25 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/A/0721721744834162a605031969826d25 2024-11-17T15:39:07,951 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 420677de373b14e08d29085670121ff4/A of 420677de373b14e08d29085670121ff4 into 0721721744834162a605031969826d25(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T15:39:07,951 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 420677de373b14e08d29085670121ff4: 2024-11-17T15:39:07,951 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., storeName=420677de373b14e08d29085670121ff4/A, priority=12, startTime=1731857947919; duration=0sec 2024-11-17T15:39:07,951 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T15:39:07,951 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 420677de373b14e08d29085670121ff4:A 2024-11-17T15:39:07,951 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T15:39:07,952 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T15:39:07,952 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1541): 420677de373b14e08d29085670121ff4/C is initiating minor compaction (all files) 2024-11-17T15:39:07,952 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 420677de373b14e08d29085670121ff4/C in TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. 2024-11-17T15:39:07,952 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/C/7ee62f6a879e45a182ca1171bfaec364, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/C/c2b293984fc445828812b5b511b8a53a, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/C/bcc5ee4ef2cd489482d25b0178860951, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/C/26677ca8f30549be9190fce2c95fbd59] into tmpdir=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp, totalSize=47.8 K 2024-11-17T15:39:07,953 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7ee62f6a879e45a182ca1171bfaec364, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1731857942586 2024-11-17T15:39:07,953 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting c2b293984fc445828812b5b511b8a53a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731857944406 2024-11-17T15:39:07,953 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting bcc5ee4ef2cd489482d25b0178860951, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1731857944476 2024-11-17T15:39:07,953 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 26677ca8f30549be9190fce2c95fbd59, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1731857946664 2024-11-17T15:39:07,961 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 420677de373b14e08d29085670121ff4#C#compaction#410 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T15:39:07,961 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/C/ea6119db37fc4e0bba6d427518cd3e8a is 50, key is test_row_0/C:col10/1731857946671/Put/seqid=0 2024-11-17T15:39:07,962 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:07,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48474 deadline: 1731858007950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:07,962 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:07,962 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:07,962 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:39:07,962 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:07,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48406 deadline: 1731858007952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:07,963 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:07,963 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:07,963 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:39:07,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742312_1488 (size=12629) 2024-11-17T15:39:07,968 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:07,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48458 deadline: 1731858007962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:07,969 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/C/ea6119db37fc4e0bba6d427518cd3e8a as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/C/ea6119db37fc4e0bba6d427518cd3e8a 2024-11-17T15:39:07,970 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:07,970 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:07,970 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:39:07,973 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 420677de373b14e08d29085670121ff4/C of 420677de373b14e08d29085670121ff4 into ea6119db37fc4e0bba6d427518cd3e8a(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T15:39:07,973 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 420677de373b14e08d29085670121ff4: 2024-11-17T15:39:07,973 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., storeName=420677de373b14e08d29085670121ff4/C, priority=12, startTime=1731857947919; duration=0sec 2024-11-17T15:39:07,973 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T15:39:07,973 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 420677de373b14e08d29085670121ff4:C 2024-11-17T15:39:08,023 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-17T15:39:08,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. 2024-11-17T15:39:08,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. as already flushing 2024-11-17T15:39:08,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. 2024-11-17T15:39:08,024 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:39:08,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:39:08,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:39:08,076 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:08,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48474 deadline: 1731858008073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:08,076 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:08,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48406 deadline: 1731858008073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:08,077 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:08,077 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:08,077 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:39:08,077 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:08,077 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:08,077 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:08,077 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:39:08,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48458 deadline: 1731858008073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:08,077 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:08,077 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:08,077 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:39:08,176 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-17T15:39:08,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. 2024-11-17T15:39:08,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. as already flushing 2024-11-17T15:39:08,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. 2024-11-17T15:39:08,176 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:39:08,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:39:08,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:39:08,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-17T15:39:08,287 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:08,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48406 deadline: 1731858008283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:08,288 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:08,288 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:08,288 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:08,288 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:39:08,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48474 deadline: 1731858008283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:08,288 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:08,288 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:08,288 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:08,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48458 deadline: 1731858008283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:08,288 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:39:08,288 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:08,288 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:08,288 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:39:08,329 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-17T15:39:08,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. 2024-11-17T15:39:08,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. as already flushing 2024-11-17T15:39:08,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. 2024-11-17T15:39:08,330 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:39:08,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:39:08,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:39:08,342 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/A/bd253feab9074b2cabea2a742df66b24 2024-11-17T15:39:08,347 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/B/1e3287eebbf24045a49d143486988d98 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/B/1e3287eebbf24045a49d143486988d98 2024-11-17T15:39:08,349 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/B/9d23c82e83874c45aec686107121ab31 is 50, key is test_row_0/B:col10/1731857946739/Put/seqid=0 2024-11-17T15:39:08,351 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 420677de373b14e08d29085670121ff4/B of 420677de373b14e08d29085670121ff4 into 1e3287eebbf24045a49d143486988d98(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T15:39:08,351 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 420677de373b14e08d29085670121ff4: 2024-11-17T15:39:08,351 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., storeName=420677de373b14e08d29085670121ff4/B, priority=12, startTime=1731857947919; duration=0sec 2024-11-17T15:39:08,351 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T15:39:08,351 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 420677de373b14e08d29085670121ff4:B 2024-11-17T15:39:08,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742313_1489 (size=12151) 2024-11-17T15:39:08,482 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-17T15:39:08,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. 2024-11-17T15:39:08,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. as already flushing 2024-11-17T15:39:08,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. 2024-11-17T15:39:08,482 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:39:08,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:39:08,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:39:08,596 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:08,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48406 deadline: 1731858008592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:08,597 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:08,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48474 deadline: 1731858008593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:08,597 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:08,597 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:08,597 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:39:08,597 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:08,597 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:08,597 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:08,597 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:39:08,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48458 deadline: 1731858008593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:08,598 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:08,598 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:08,598 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:39:08,635 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-17T15:39:08,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. 2024-11-17T15:39:08,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. as already flushing 2024-11-17T15:39:08,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. 2024-11-17T15:39:08,635 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:39:08,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:39:08,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:39:08,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-17T15:39:08,754 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/B/9d23c82e83874c45aec686107121ab31 2024-11-17T15:39:08,761 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/C/1c4c5e3c44d440ceac65d9b48b26f38a is 50, key is test_row_0/C:col10/1731857946739/Put/seqid=0 2024-11-17T15:39:08,764 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742314_1490 (size=12151) 2024-11-17T15:39:08,788 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-17T15:39:08,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. 2024-11-17T15:39:08,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. as already flushing 2024-11-17T15:39:08,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. 2024-11-17T15:39:08,788 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:39:08,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:39:08,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:39:08,940 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-17T15:39:08,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. 2024-11-17T15:39:08,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. as already flushing 2024-11-17T15:39:08,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. 2024-11-17T15:39:08,941 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:39:08,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:39:08,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:39:09,091 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-17T15:39:09,093 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-17T15:39:09,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. 2024-11-17T15:39:09,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. as already flushing 2024-11-17T15:39:09,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. 2024-11-17T15:39:09,094 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:39:09,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:39:09,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:39:09,106 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:09,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48406 deadline: 1731858009103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:09,106 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:09,107 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:09,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48474 deadline: 1731858009103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:09,107 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:09,107 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:39:09,107 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:09,107 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:09,107 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:09,107 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:39:09,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48458 deadline: 1731858009103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:09,107 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:09,107 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:09,107 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:39:09,165 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/C/1c4c5e3c44d440ceac65d9b48b26f38a 2024-11-17T15:39:09,169 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/A/bd253feab9074b2cabea2a742df66b24 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/A/bd253feab9074b2cabea2a742df66b24 2024-11-17T15:39:09,175 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/A/bd253feab9074b2cabea2a742df66b24, entries=150, sequenceid=235, filesize=11.9 K 2024-11-17T15:39:09,175 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/B/9d23c82e83874c45aec686107121ab31 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/B/9d23c82e83874c45aec686107121ab31 2024-11-17T15:39:09,178 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/B/9d23c82e83874c45aec686107121ab31, entries=150, sequenceid=235, filesize=11.9 K 2024-11-17T15:39:09,179 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/C/1c4c5e3c44d440ceac65d9b48b26f38a as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/C/1c4c5e3c44d440ceac65d9b48b26f38a 2024-11-17T15:39:09,182 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/C/1c4c5e3c44d440ceac65d9b48b26f38a, entries=150, sequenceid=235, filesize=11.9 K 2024-11-17T15:39:09,182 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for 420677de373b14e08d29085670121ff4 in 1250ms, sequenceid=235, compaction requested=false 2024-11-17T15:39:09,183 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 420677de373b14e08d29085670121ff4: 2024-11-17T15:39:09,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(8855): Flush requested on 420677de373b14e08d29085670121ff4 2024-11-17T15:39:09,231 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T15:39:09,231 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 420677de373b14e08d29085670121ff4 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-17T15:39:09,235 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/A/63eb6d0797b746799e9924fe880b94da is 50, key is test_row_0/A:col10/1731857947961/Put/seqid=0 2024-11-17T15:39:09,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742315_1491 (size=14541) 2024-11-17T15:39:09,246 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-17T15:39:09,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. 2024-11-17T15:39:09,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. as already flushing 2024-11-17T15:39:09,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. 2024-11-17T15:39:09,247 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:39:09,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:39:09,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:39:09,399 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-17T15:39:09,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. 2024-11-17T15:39:09,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. as already flushing 2024-11-17T15:39:09,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. 2024-11-17T15:39:09,399 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:39:09,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:39:09,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:39:09,401 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:09,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48390 deadline: 1731858009396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:09,401 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:09,401 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:09,401 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:39:09,512 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:09,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48390 deadline: 1731858009503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:09,512 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:09,512 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:09,512 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:39:09,552 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-17T15:39:09,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. 2024-11-17T15:39:09,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. as already flushing 2024-11-17T15:39:09,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. 2024-11-17T15:39:09,553 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:39:09,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:39:09,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:39:09,642 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/A/63eb6d0797b746799e9924fe880b94da 2024-11-17T15:39:09,648 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/B/ab9fddf0e1d64b489a716ace383c7558 is 50, key is test_row_0/B:col10/1731857947961/Put/seqid=0 2024-11-17T15:39:09,652 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742316_1492 (size=12151) 2024-11-17T15:39:09,704 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-17T15:39:09,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. 2024-11-17T15:39:09,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. as already flushing 2024-11-17T15:39:09,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. 2024-11-17T15:39:09,705 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:39:09,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:39:09,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:39:09,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-17T15:39:09,729 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:09,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48390 deadline: 1731858009722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:09,729 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:09,729 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:09,729 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:39:09,857 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-17T15:39:09,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. 2024-11-17T15:39:09,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. as already flushing 2024-11-17T15:39:09,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. 2024-11-17T15:39:09,858 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:39:09,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:39:09,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:39:10,010 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-17T15:39:10,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. 2024-11-17T15:39:10,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. as already flushing 2024-11-17T15:39:10,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. 2024-11-17T15:39:10,011 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:39:10,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:39:10,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:39:10,038 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:10,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48390 deadline: 1731858010033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:10,038 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:10,038 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:10,038 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:39:10,052 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/B/ab9fddf0e1d64b489a716ace383c7558 2024-11-17T15:39:10,058 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/C/13469041a4974862a4869a04aa3c37e2 is 50, key is test_row_0/C:col10/1731857947961/Put/seqid=0 2024-11-17T15:39:10,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742317_1493 (size=12151) 2024-11-17T15:39:10,131 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:10,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48406 deadline: 1731858010122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:10,132 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:10,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48474 deadline: 1731858010123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:10,132 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:10,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48458 deadline: 1731858010123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:10,132 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 7a780d55532c,32955,1731857800648 for 'test_row_1' in 420677de373b14e08d29085670121ff4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2180 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T15:39:10,132 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:10,132 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:10,132 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:39:10,133 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 7a780d55532c,32955,1731857800648 for 'test_row_2' in 420677de373b14e08d29085670121ff4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2170 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T15:39:10,133 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:10,133 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:10,133 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:39:10,133 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 7a780d55532c,32955,1731857800648 for 'test_row_0' in 420677de373b14e08d29085670121ff4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2185 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T15:39:10,133 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:10,133 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:10,133 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:39:10,163 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-17T15:39:10,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. 2024-11-17T15:39:10,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. as already flushing 2024-11-17T15:39:10,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. 2024-11-17T15:39:10,164 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:39:10,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:39:10,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:39:10,316 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-17T15:39:10,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. 2024-11-17T15:39:10,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. as already flushing 2024-11-17T15:39:10,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. 2024-11-17T15:39:10,316 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:39:10,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:39:10,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:39:10,462 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/C/13469041a4974862a4869a04aa3c37e2 2024-11-17T15:39:10,466 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/A/63eb6d0797b746799e9924fe880b94da as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/A/63eb6d0797b746799e9924fe880b94da 2024-11-17T15:39:10,468 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-17T15:39:10,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. 2024-11-17T15:39:10,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. as already flushing 2024-11-17T15:39:10,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. 2024-11-17T15:39:10,469 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:39:10,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:39:10,469 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/A/63eb6d0797b746799e9924fe880b94da, entries=200, sequenceid=249, filesize=14.2 K 2024-11-17T15:39:10,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:39:10,470 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/B/ab9fddf0e1d64b489a716ace383c7558 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/B/ab9fddf0e1d64b489a716ace383c7558 2024-11-17T15:39:10,474 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/B/ab9fddf0e1d64b489a716ace383c7558, entries=150, sequenceid=249, filesize=11.9 K 2024-11-17T15:39:10,474 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/C/13469041a4974862a4869a04aa3c37e2 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/C/13469041a4974862a4869a04aa3c37e2 2024-11-17T15:39:10,477 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/C/13469041a4974862a4869a04aa3c37e2, entries=150, sequenceid=249, filesize=11.9 K 2024-11-17T15:39:10,477 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 420677de373b14e08d29085670121ff4 in 1246ms, sequenceid=249, compaction requested=true 2024-11-17T15:39:10,478 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 420677de373b14e08d29085670121ff4: 2024-11-17T15:39:10,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 420677de373b14e08d29085670121ff4:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T15:39:10,478 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T15:39:10,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T15:39:10,478 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T15:39:10,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 420677de373b14e08d29085670121ff4:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T15:39:10,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T15:39:10,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 420677de373b14e08d29085670121ff4:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T15:39:10,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T15:39:10,479 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T15:39:10,479 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39321 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T15:39:10,479 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1541): 420677de373b14e08d29085670121ff4/A is initiating minor compaction (all files) 2024-11-17T15:39:10,479 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1541): 420677de373b14e08d29085670121ff4/B is initiating minor compaction (all files) 2024-11-17T15:39:10,479 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 420677de373b14e08d29085670121ff4/B in TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. 2024-11-17T15:39:10,479 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 420677de373b14e08d29085670121ff4/A in TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. 2024-11-17T15:39:10,479 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/B/1e3287eebbf24045a49d143486988d98, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/B/9d23c82e83874c45aec686107121ab31, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/B/ab9fddf0e1d64b489a716ace383c7558] into tmpdir=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp, totalSize=36.1 K 2024-11-17T15:39:10,479 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/A/0721721744834162a605031969826d25, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/A/bd253feab9074b2cabea2a742df66b24, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/A/63eb6d0797b746799e9924fe880b94da] into tmpdir=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp, totalSize=38.4 K 2024-11-17T15:39:10,480 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0721721744834162a605031969826d25, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1731857946664 2024-11-17T15:39:10,480 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting 1e3287eebbf24045a49d143486988d98, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1731857946664 2024-11-17T15:39:10,480 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting 9d23c82e83874c45aec686107121ab31, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1731857946737 2024-11-17T15:39:10,480 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting bd253feab9074b2cabea2a742df66b24, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1731857946737 2024-11-17T15:39:10,480 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 63eb6d0797b746799e9924fe880b94da, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1731857947948 2024-11-17T15:39:10,480 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting ab9fddf0e1d64b489a716ace383c7558, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1731857947961 2024-11-17T15:39:10,486 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 420677de373b14e08d29085670121ff4#A#compaction#416 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T15:39:10,487 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/A/b198e30eceaa4e879ad52a0c747d2a4a is 50, key is test_row_0/A:col10/1731857947961/Put/seqid=0 2024-11-17T15:39:10,487 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 420677de373b14e08d29085670121ff4#B#compaction#417 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T15:39:10,487 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/B/8f45c7be662d4b47b61acd4f67767e6e is 50, key is test_row_0/B:col10/1731857947961/Put/seqid=0 2024-11-17T15:39:10,499 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742318_1494 (size=12731) 2024-11-17T15:39:10,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742319_1495 (size=12731) 2024-11-17T15:39:10,504 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/A/b198e30eceaa4e879ad52a0c747d2a4a as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/A/b198e30eceaa4e879ad52a0c747d2a4a 2024-11-17T15:39:10,506 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/B/8f45c7be662d4b47b61acd4f67767e6e as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/B/8f45c7be662d4b47b61acd4f67767e6e 2024-11-17T15:39:10,511 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 420677de373b14e08d29085670121ff4/A of 420677de373b14e08d29085670121ff4 into b198e30eceaa4e879ad52a0c747d2a4a(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T15:39:10,511 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 420677de373b14e08d29085670121ff4: 2024-11-17T15:39:10,511 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., storeName=420677de373b14e08d29085670121ff4/A, priority=13, startTime=1731857950478; duration=0sec 2024-11-17T15:39:10,511 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T15:39:10,511 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 420677de373b14e08d29085670121ff4:A 2024-11-17T15:39:10,511 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T15:39:10,512 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 420677de373b14e08d29085670121ff4/B of 420677de373b14e08d29085670121ff4 into 8f45c7be662d4b47b61acd4f67767e6e(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T15:39:10,512 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 420677de373b14e08d29085670121ff4: 2024-11-17T15:39:10,512 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., storeName=420677de373b14e08d29085670121ff4/B, priority=13, startTime=1731857950478; duration=0sec 2024-11-17T15:39:10,512 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T15:39:10,512 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T15:39:10,512 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1541): 420677de373b14e08d29085670121ff4/C is initiating minor compaction (all files) 2024-11-17T15:39:10,512 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 420677de373b14e08d29085670121ff4:B 2024-11-17T15:39:10,512 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 420677de373b14e08d29085670121ff4/C in TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. 2024-11-17T15:39:10,513 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/C/ea6119db37fc4e0bba6d427518cd3e8a, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/C/1c4c5e3c44d440ceac65d9b48b26f38a, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/C/13469041a4974862a4869a04aa3c37e2] into tmpdir=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp, totalSize=36.1 K 2024-11-17T15:39:10,513 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting ea6119db37fc4e0bba6d427518cd3e8a, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1731857946664 2024-11-17T15:39:10,513 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1c4c5e3c44d440ceac65d9b48b26f38a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1731857946737 2024-11-17T15:39:10,513 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 13469041a4974862a4869a04aa3c37e2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1731857947961 2024-11-17T15:39:10,521 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 420677de373b14e08d29085670121ff4#C#compaction#418 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T15:39:10,522 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/C/b1686b64b37549daa2e400cb16667353 is 50, key is test_row_0/C:col10/1731857947961/Put/seqid=0 2024-11-17T15:39:10,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742320_1496 (size=12731) 2024-11-17T15:39:10,536 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/C/b1686b64b37549daa2e400cb16667353 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/C/b1686b64b37549daa2e400cb16667353 2024-11-17T15:39:10,541 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 420677de373b14e08d29085670121ff4/C of 420677de373b14e08d29085670121ff4 into b1686b64b37549daa2e400cb16667353(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T15:39:10,541 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 420677de373b14e08d29085670121ff4: 2024-11-17T15:39:10,541 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., storeName=420677de373b14e08d29085670121ff4/C, priority=13, startTime=1731857950478; duration=0sec 2024-11-17T15:39:10,541 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T15:39:10,541 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 420677de373b14e08d29085670121ff4:C 2024-11-17T15:39:10,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(8855): Flush requested on 420677de373b14e08d29085670121ff4 2024-11-17T15:39:10,548 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T15:39:10,548 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 420677de373b14e08d29085670121ff4 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-17T15:39:10,553 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/A/be48caf0e9ea402ebcadc08889269990 is 50, key is test_row_0/A:col10/1731857950547/Put/seqid=0 2024-11-17T15:39:10,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742321_1497 (size=14741) 2024-11-17T15:39:10,561 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=278 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/A/be48caf0e9ea402ebcadc08889269990 2024-11-17T15:39:10,571 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/B/b6f5e8c4cb7040f0856dc79bc819bdb1 is 50, key is test_row_0/B:col10/1731857950547/Put/seqid=0 2024-11-17T15:39:10,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742322_1498 (size=12301) 2024-11-17T15:39:10,577 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=278 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/B/b6f5e8c4cb7040f0856dc79bc819bdb1 2024-11-17T15:39:10,584 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/C/04bc1014130144aba688ba0644a0aded is 50, key is test_row_0/C:col10/1731857950547/Put/seqid=0 2024-11-17T15:39:10,599 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742323_1499 (size=12301) 2024-11-17T15:39:10,599 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=278 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/C/04bc1014130144aba688ba0644a0aded 2024-11-17T15:39:10,603 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/A/be48caf0e9ea402ebcadc08889269990 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/A/be48caf0e9ea402ebcadc08889269990 2024-11-17T15:39:10,606 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/A/be48caf0e9ea402ebcadc08889269990, entries=200, sequenceid=278, filesize=14.4 K 2024-11-17T15:39:10,608 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/B/b6f5e8c4cb7040f0856dc79bc819bdb1 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/B/b6f5e8c4cb7040f0856dc79bc819bdb1 2024-11-17T15:39:10,611 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/B/b6f5e8c4cb7040f0856dc79bc819bdb1, entries=150, sequenceid=278, filesize=12.0 K 2024-11-17T15:39:10,611 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/C/04bc1014130144aba688ba0644a0aded as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/C/04bc1014130144aba688ba0644a0aded 2024-11-17T15:39:10,612 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:10,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48390 deadline: 1731858010603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:10,613 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:10,613 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:10,613 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:39:10,618 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/C/04bc1014130144aba688ba0644a0aded, entries=150, sequenceid=278, filesize=12.0 K 2024-11-17T15:39:10,619 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 420677de373b14e08d29085670121ff4 in 70ms, sequenceid=278, compaction requested=false 2024-11-17T15:39:10,619 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 420677de373b14e08d29085670121ff4: 2024-11-17T15:39:10,625 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-17T15:39:10,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. 2024-11-17T15:39:10,625 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2902): Flushing 420677de373b14e08d29085670121ff4 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-17T15:39:10,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/A/64c2d140fac44f5f8829bad8dec76694 is 50, key is test_row_0/A:col10/1731857950591/Put/seqid=0 2024-11-17T15:39:10,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742324_1500 (size=12301) 2024-11-17T15:39:10,651 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/A/64c2d140fac44f5f8829bad8dec76694 2024-11-17T15:39:10,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/B/2314b67e48464084a68b27d4cb08de46 is 50, key is test_row_0/B:col10/1731857950591/Put/seqid=0 2024-11-17T15:39:10,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742325_1501 (size=12301) 2024-11-17T15:39:10,670 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/B/2314b67e48464084a68b27d4cb08de46 2024-11-17T15:39:10,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/C/ca3df9f84ef148d298c3a4faf492cb38 is 50, key is test_row_0/C:col10/1731857950591/Put/seqid=0 2024-11-17T15:39:10,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742326_1502 (size=12301) 2024-11-17T15:39:10,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(8855): Flush requested on 420677de373b14e08d29085670121ff4 2024-11-17T15:39:10,727 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. as already flushing 2024-11-17T15:39:10,930 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:10,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48390 deadline: 1731858010917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:10,931 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:10,931 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:10,931 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:39:11,041 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:11,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48390 deadline: 1731858011033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:11,041 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:11,041 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:11,041 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:39:11,087 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/C/ca3df9f84ef148d298c3a4faf492cb38 2024-11-17T15:39:11,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/A/64c2d140fac44f5f8829bad8dec76694 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/A/64c2d140fac44f5f8829bad8dec76694 2024-11-17T15:39:11,094 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/A/64c2d140fac44f5f8829bad8dec76694, entries=150, sequenceid=288, filesize=12.0 K 2024-11-17T15:39:11,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/B/2314b67e48464084a68b27d4cb08de46 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/B/2314b67e48464084a68b27d4cb08de46 2024-11-17T15:39:11,099 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/B/2314b67e48464084a68b27d4cb08de46, entries=150, sequenceid=288, filesize=12.0 K 2024-11-17T15:39:11,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/C/ca3df9f84ef148d298c3a4faf492cb38 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/C/ca3df9f84ef148d298c3a4faf492cb38 2024-11-17T15:39:11,102 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/C/ca3df9f84ef148d298c3a4faf492cb38, entries=150, sequenceid=288, filesize=12.0 K 2024-11-17T15:39:11,103 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=154.31 KB/158010 for 420677de373b14e08d29085670121ff4 in 478ms, sequenceid=288, compaction requested=true 2024-11-17T15:39:11,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2603): Flush status journal for 420677de373b14e08d29085670121ff4: 2024-11-17T15:39:11,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. 2024-11-17T15:39:11,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=132 2024-11-17T15:39:11,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.HMaster(4169): Remote procedure done, pid=132 2024-11-17T15:39:11,107 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-11-17T15:39:11,107 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.5390 sec 2024-11-17T15:39:11,109 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees in 3.5450 sec 2024-11-17T15:39:11,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(8855): Flush requested on 420677de373b14e08d29085670121ff4 2024-11-17T15:39:11,248 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T15:39:11,248 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 420677de373b14e08d29085670121ff4 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-17T15:39:11,253 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/A/983081ef739f484ebe035b423402432e is 50, key is test_row_0/A:col10/1731857950804/Put/seqid=0 2024-11-17T15:39:11,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742327_1503 (size=14741) 2024-11-17T15:39:11,258 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=315 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/A/983081ef739f484ebe035b423402432e 2024-11-17T15:39:11,265 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/B/723fc23987244c0681bd591019641aa3 is 50, key is test_row_0/B:col10/1731857950804/Put/seqid=0 2024-11-17T15:39:11,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742328_1504 (size=12301) 2024-11-17T15:39:11,268 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=315 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/B/723fc23987244c0681bd591019641aa3 2024-11-17T15:39:11,274 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/C/0913d564e47b417bbb66f007ab8781fd is 50, key is test_row_0/C:col10/1731857950804/Put/seqid=0 2024-11-17T15:39:11,276 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742329_1505 (size=12301) 2024-11-17T15:39:11,277 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=315 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/C/0913d564e47b417bbb66f007ab8781fd 2024-11-17T15:39:11,280 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/A/983081ef739f484ebe035b423402432e as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/A/983081ef739f484ebe035b423402432e 2024-11-17T15:39:11,284 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/A/983081ef739f484ebe035b423402432e, entries=200, sequenceid=315, filesize=14.4 K 2024-11-17T15:39:11,285 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/B/723fc23987244c0681bd591019641aa3 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/B/723fc23987244c0681bd591019641aa3 2024-11-17T15:39:11,288 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/B/723fc23987244c0681bd591019641aa3, entries=150, sequenceid=315, filesize=12.0 K 2024-11-17T15:39:11,288 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/C/0913d564e47b417bbb66f007ab8781fd as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/C/0913d564e47b417bbb66f007ab8781fd 2024-11-17T15:39:11,291 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/C/0913d564e47b417bbb66f007ab8781fd, entries=150, sequenceid=315, filesize=12.0 K 2024-11-17T15:39:11,292 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=33.54 KB/34350 for 420677de373b14e08d29085670121ff4 in 44ms, sequenceid=315, compaction requested=true 2024-11-17T15:39:11,292 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 420677de373b14e08d29085670121ff4: 2024-11-17T15:39:11,292 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 420677de373b14e08d29085670121ff4:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T15:39:11,292 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T15:39:11,292 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 420677de373b14e08d29085670121ff4:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T15:39:11,292 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T15:39:11,292 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T15:39:11,292 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 420677de373b14e08d29085670121ff4:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T15:39:11,292 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T15:39:11,292 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T15:39:11,294 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54514 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T15:39:11,294 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49634 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T15:39:11,294 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1541): 420677de373b14e08d29085670121ff4/A is initiating minor compaction (all files) 2024-11-17T15:39:11,294 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1541): 420677de373b14e08d29085670121ff4/B is initiating minor compaction (all files) 2024-11-17T15:39:11,294 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 420677de373b14e08d29085670121ff4/A in TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. 2024-11-17T15:39:11,294 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 420677de373b14e08d29085670121ff4/B in TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. 2024-11-17T15:39:11,294 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/A/b198e30eceaa4e879ad52a0c747d2a4a, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/A/be48caf0e9ea402ebcadc08889269990, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/A/64c2d140fac44f5f8829bad8dec76694, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/A/983081ef739f484ebe035b423402432e] into tmpdir=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp, totalSize=53.2 K 2024-11-17T15:39:11,294 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/B/8f45c7be662d4b47b61acd4f67767e6e, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/B/b6f5e8c4cb7040f0856dc79bc819bdb1, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/B/2314b67e48464084a68b27d4cb08de46, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/B/723fc23987244c0681bd591019641aa3] into tmpdir=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp, totalSize=48.5 K 2024-11-17T15:39:11,294 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting b198e30eceaa4e879ad52a0c747d2a4a, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1731857947961 2024-11-17T15:39:11,294 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting 8f45c7be662d4b47b61acd4f67767e6e, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1731857947961 2024-11-17T15:39:11,295 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting be48caf0e9ea402ebcadc08889269990, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1731857949365 2024-11-17T15:39:11,295 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting b6f5e8c4cb7040f0856dc79bc819bdb1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1731857949372 2024-11-17T15:39:11,295 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting 2314b67e48464084a68b27d4cb08de46, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1731857950583 2024-11-17T15:39:11,295 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 64c2d140fac44f5f8829bad8dec76694, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1731857950583 2024-11-17T15:39:11,295 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting 723fc23987244c0681bd591019641aa3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1731857950804 2024-11-17T15:39:11,295 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 983081ef739f484ebe035b423402432e, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1731857950804 2024-11-17T15:39:11,303 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 420677de373b14e08d29085670121ff4#B#compaction#428 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T15:39:11,304 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 420677de373b14e08d29085670121ff4#A#compaction#429 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T15:39:11,304 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/B/a185eb8574c443faa7e5500b5c2b3ff5 is 50, key is test_row_0/B:col10/1731857950804/Put/seqid=0 2024-11-17T15:39:11,304 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/A/b5331c1409a34124b7b76601f691cdbd is 50, key is test_row_0/A:col10/1731857950804/Put/seqid=0 2024-11-17T15:39:11,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(8855): Flush requested on 420677de373b14e08d29085670121ff4 2024-11-17T15:39:11,311 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T15:39:11,311 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 420677de373b14e08d29085670121ff4 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-17T15:39:11,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742330_1506 (size=13017) 2024-11-17T15:39:11,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742331_1507 (size=13017) 2024-11-17T15:39:11,321 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/A/f81903ead88b4fd0a310201fb06d3beb is 50, key is test_row_0/A:col10/1731857951296/Put/seqid=0 2024-11-17T15:39:11,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742332_1508 (size=14741) 2024-11-17T15:39:11,514 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:11,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48390 deadline: 1731858011508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:11,514 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:11,514 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:11,514 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:39:11,633 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:11,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48390 deadline: 1731858011623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:11,633 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:11,633 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:11,633 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:39:11,724 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/B/a185eb8574c443faa7e5500b5c2b3ff5 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/B/a185eb8574c443faa7e5500b5c2b3ff5 2024-11-17T15:39:11,725 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/A/b5331c1409a34124b7b76601f691cdbd as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/A/b5331c1409a34124b7b76601f691cdbd 2024-11-17T15:39:11,728 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 420677de373b14e08d29085670121ff4/A of 420677de373b14e08d29085670121ff4 into b5331c1409a34124b7b76601f691cdbd(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T15:39:11,728 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 420677de373b14e08d29085670121ff4: 2024-11-17T15:39:11,728 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., storeName=420677de373b14e08d29085670121ff4/A, priority=12, startTime=1731857951292; duration=0sec 2024-11-17T15:39:11,728 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T15:39:11,728 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 420677de373b14e08d29085670121ff4:A 2024-11-17T15:39:11,728 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T15:39:11,729 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 420677de373b14e08d29085670121ff4/B of 420677de373b14e08d29085670121ff4 into a185eb8574c443faa7e5500b5c2b3ff5(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T15:39:11,729 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 420677de373b14e08d29085670121ff4: 2024-11-17T15:39:11,729 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., storeName=420677de373b14e08d29085670121ff4/B, priority=12, startTime=1731857951292; duration=0sec 2024-11-17T15:39:11,729 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T15:39:11,729 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 420677de373b14e08d29085670121ff4:B 2024-11-17T15:39:11,729 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49634 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T15:39:11,730 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1541): 420677de373b14e08d29085670121ff4/C is initiating minor compaction (all files) 2024-11-17T15:39:11,730 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 420677de373b14e08d29085670121ff4/C in TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. 2024-11-17T15:39:11,730 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/C/b1686b64b37549daa2e400cb16667353, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/C/04bc1014130144aba688ba0644a0aded, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/C/ca3df9f84ef148d298c3a4faf492cb38, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/C/0913d564e47b417bbb66f007ab8781fd] into tmpdir=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp, totalSize=48.5 K 2024-11-17T15:39:11,730 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting b1686b64b37549daa2e400cb16667353, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1731857947961 2024-11-17T15:39:11,730 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 04bc1014130144aba688ba0644a0aded, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1731857949372 2024-11-17T15:39:11,730 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting ca3df9f84ef148d298c3a4faf492cb38, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1731857950583 2024-11-17T15:39:11,731 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0913d564e47b417bbb66f007ab8781fd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1731857950804 2024-11-17T15:39:11,732 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/A/f81903ead88b4fd0a310201fb06d3beb 2024-11-17T15:39:11,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-17T15:39:11,734 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T15:39:11,734 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T15:39:11,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees 2024-11-17T15:39:11,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-17T15:39:11,736 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T15:39:11,737 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T15:39:11,737 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T15:39:11,740 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/B/f4abf2ba24034ff4935d2b7e9b0d1510 is 50, key is test_row_0/B:col10/1731857951296/Put/seqid=0 2024-11-17T15:39:11,740 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 420677de373b14e08d29085670121ff4#C#compaction#431 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T15:39:11,741 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/C/6a4b3958b11f4403a683feef1bf61c6f is 50, key is test_row_0/C:col10/1731857950804/Put/seqid=0 2024-11-17T15:39:11,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742334_1510 (size=13017) 2024-11-17T15:39:11,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742333_1509 (size=12301) 2024-11-17T15:39:11,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-17T15:39:11,851 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:11,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48390 deadline: 1731858011843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:11,852 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:11,852 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:11,852 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:39:11,891 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-17T15:39:11,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. 2024-11-17T15:39:11,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. as already flushing 2024-11-17T15:39:11,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. 2024-11-17T15:39:11,892 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:39:11,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:39:11,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:39:12,044 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-17T15:39:12,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. 2024-11-17T15:39:12,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. as already flushing 2024-11-17T15:39:12,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. 2024-11-17T15:39:12,045 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:39:12,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:39:12,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:39:12,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-17T15:39:12,148 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:12,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48458 deadline: 1731858012143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:12,148 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/C/6a4b3958b11f4403a683feef1bf61c6f as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/C/6a4b3958b11f4403a683feef1bf61c6f 2024-11-17T15:39:12,148 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 7a780d55532c,32955,1731857800648 for 'test_row_2' in 420677de373b14e08d29085670121ff4 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4186 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T15:39:12,149 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:12,149 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:12,149 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:39:12,150 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/B/f4abf2ba24034ff4935d2b7e9b0d1510 2024-11-17T15:39:12,153 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 420677de373b14e08d29085670121ff4/C of 420677de373b14e08d29085670121ff4 into 6a4b3958b11f4403a683feef1bf61c6f(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T15:39:12,153 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 420677de373b14e08d29085670121ff4: 2024-11-17T15:39:12,153 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., storeName=420677de373b14e08d29085670121ff4/C, priority=12, startTime=1731857951292; duration=0sec 2024-11-17T15:39:12,153 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T15:39:12,153 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 420677de373b14e08d29085670121ff4:C 2024-11-17T15:39:12,156 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/C/287ba7a9bf2948ea8c020c2cac89edd5 is 50, key is test_row_0/C:col10/1731857951296/Put/seqid=0 2024-11-17T15:39:12,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742335_1511 (size=12301) 2024-11-17T15:39:12,161 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:12,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48406 deadline: 1731858012153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:12,161 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 7a780d55532c,32955,1731857800648 for 'test_row_1' in 420677de373b14e08d29085670121ff4 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4210 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T15:39:12,162 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:12,162 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:12,162 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:39:12,173 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:12,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48474 deadline: 1731858012163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:12,174 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:12,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48390 deadline: 1731858012163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:12,174 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 7a780d55532c,32955,1731857800648 for 'test_row_0' in 420677de373b14e08d29085670121ff4 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4226 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T15:39:12,174 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:12,174 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:12,174 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:12,174 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:12,174 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:39:12,174 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:39:12,197 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-17T15:39:12,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. 2024-11-17T15:39:12,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. as already flushing 2024-11-17T15:39:12,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. 2024-11-17T15:39:12,197 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:39:12,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:39:12,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:39:12,350 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-17T15:39:12,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. 2024-11-17T15:39:12,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. as already flushing 2024-11-17T15:39:12,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. 2024-11-17T15:39:12,350 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:39:12,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:39:12,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:39:12,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-17T15:39:12,502 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-17T15:39:12,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. 2024-11-17T15:39:12,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. as already flushing 2024-11-17T15:39:12,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. 2024-11-17T15:39:12,503 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:39:12,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:39:12,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:39:12,560 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/C/287ba7a9bf2948ea8c020c2cac89edd5 2024-11-17T15:39:12,563 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/A/f81903ead88b4fd0a310201fb06d3beb as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/A/f81903ead88b4fd0a310201fb06d3beb 2024-11-17T15:39:12,566 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/A/f81903ead88b4fd0a310201fb06d3beb, entries=200, sequenceid=326, filesize=14.4 K 2024-11-17T15:39:12,567 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/B/f4abf2ba24034ff4935d2b7e9b0d1510 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/B/f4abf2ba24034ff4935d2b7e9b0d1510 2024-11-17T15:39:12,570 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/B/f4abf2ba24034ff4935d2b7e9b0d1510, entries=150, sequenceid=326, filesize=12.0 K 2024-11-17T15:39:12,571 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/C/287ba7a9bf2948ea8c020c2cac89edd5 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/C/287ba7a9bf2948ea8c020c2cac89edd5 2024-11-17T15:39:12,573 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/C/287ba7a9bf2948ea8c020c2cac89edd5, entries=150, sequenceid=326, filesize=12.0 K 2024-11-17T15:39:12,574 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 420677de373b14e08d29085670121ff4 in 1263ms, sequenceid=326, compaction requested=false 2024-11-17T15:39:12,574 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 420677de373b14e08d29085670121ff4: 2024-11-17T15:39:12,655 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-17T15:39:12,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. 2024-11-17T15:39:12,656 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2902): Flushing 420677de373b14e08d29085670121ff4 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-17T15:39:12,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/A/079950a6805746e5879bc86acc82208e is 50, key is test_row_0/A:col10/1731857951506/Put/seqid=0 2024-11-17T15:39:12,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742336_1512 (size=12301) 2024-11-17T15:39:12,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(8855): Flush requested on 420677de373b14e08d29085670121ff4 2024-11-17T15:39:12,687 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. as already flushing 2024-11-17T15:39:12,752 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:12,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48390 deadline: 1731858012742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:12,752 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:12,752 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:12,752 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:39:12,866 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:12,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48390 deadline: 1731858012863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:12,866 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:12,866 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:12,866 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:39:12,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-17T15:39:13,065 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=354 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/A/079950a6805746e5879bc86acc82208e 2024-11-17T15:39:13,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/B/e8267f0a4da340b084f80c35ce7fd8a1 is 50, key is test_row_0/B:col10/1731857951506/Put/seqid=0 2024-11-17T15:39:13,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742337_1513 (size=12301) 2024-11-17T15:39:13,076 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=354 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/B/e8267f0a4da340b084f80c35ce7fd8a1 2024-11-17T15:39:13,080 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:13,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48390 deadline: 1731858013073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:13,081 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:13,081 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:13,081 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:39:13,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/C/14a8b35cc74a44e79374c243455e29c5 is 50, key is test_row_0/C:col10/1731857951506/Put/seqid=0 2024-11-17T15:39:13,092 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742338_1514 (size=12301) 2024-11-17T15:39:13,369 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:13,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48442 deadline: 1731858013363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:13,369 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 7a780d55532c,32955,1731857800648 for 'test_row_0' in 420677de373b14e08d29085670121ff4 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18343 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T15:39:13,370 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:13,370 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:13,370 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:39:13,400 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:13,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48390 deadline: 1731858013393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:13,400 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:13,400 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:13,400 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:39:13,493 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=354 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/C/14a8b35cc74a44e79374c243455e29c5 2024-11-17T15:39:13,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/A/079950a6805746e5879bc86acc82208e as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/A/079950a6805746e5879bc86acc82208e 2024-11-17T15:39:13,501 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/A/079950a6805746e5879bc86acc82208e, entries=150, sequenceid=354, filesize=12.0 K 2024-11-17T15:39:13,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/B/e8267f0a4da340b084f80c35ce7fd8a1 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/B/e8267f0a4da340b084f80c35ce7fd8a1 2024-11-17T15:39:13,504 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/B/e8267f0a4da340b084f80c35ce7fd8a1, entries=150, sequenceid=354, filesize=12.0 K 2024-11-17T15:39:13,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/C/14a8b35cc74a44e79374c243455e29c5 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/C/14a8b35cc74a44e79374c243455e29c5 2024-11-17T15:39:13,509 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/C/14a8b35cc74a44e79374c243455e29c5, entries=150, sequenceid=354, filesize=12.0 K 2024-11-17T15:39:13,509 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 420677de373b14e08d29085670121ff4 in 854ms, sequenceid=354, compaction requested=true 2024-11-17T15:39:13,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2603): Flush status journal for 420677de373b14e08d29085670121ff4: 2024-11-17T15:39:13,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. 2024-11-17T15:39:13,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=134 2024-11-17T15:39:13,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.HMaster(4169): Remote procedure done, pid=134 2024-11-17T15:39:13,512 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-11-17T15:39:13,513 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7740 sec 2024-11-17T15:39:13,515 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees in 1.7790 sec 2024-11-17T15:39:13,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-17T15:39:13,884 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T15:39:13,884 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T15:39:13,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] procedure2.ProcedureExecutor(1139): Stored pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees 2024-11-17T15:39:13,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-17T15:39:13,885 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T15:39:13,886 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T15:39:13,886 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T15:39:13,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(8855): Flush requested on 420677de373b14e08d29085670121ff4 2024-11-17T15:39:13,918 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T15:39:13,918 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 420677de373b14e08d29085670121ff4 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-17T15:39:13,921 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/A/1f25571fc01440a7926d56873cd7afe8 is 50, key is test_row_0/A:col10/1731857952688/Put/seqid=0 2024-11-17T15:39:13,931 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742339_1515 (size=14741) 2024-11-17T15:39:13,935 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=366 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/A/1f25571fc01440a7926d56873cd7afe8 2024-11-17T15:39:13,942 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/B/a9a58f1d83f743ce806bd1ea2256fc7f is 50, key is test_row_0/B:col10/1731857952688/Put/seqid=0 2024-11-17T15:39:13,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742340_1516 (size=12301) 2024-11-17T15:39:13,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-17T15:39:14,038 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-17T15:39:14,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. 2024-11-17T15:39:14,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. as already flushing 2024-11-17T15:39:14,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. 2024-11-17T15:39:14,039 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:39:14,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:39:14,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:39:14,105 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:14,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48390 deadline: 1731858014094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:14,106 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:14,106 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:14,106 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:39:14,191 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-17T15:39:14,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. 2024-11-17T15:39:14,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. as already flushing 2024-11-17T15:39:14,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. 2024-11-17T15:39:14,191 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:39:14,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:39:14,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:39:14,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-17T15:39:14,219 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:14,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48390 deadline: 1731858014213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:14,220 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:14,220 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:14,220 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:39:14,344 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-17T15:39:14,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. 2024-11-17T15:39:14,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. as already flushing 2024-11-17T15:39:14,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. 2024-11-17T15:39:14,344 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:39:14,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:39:14,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:39:14,353 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=366 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/B/a9a58f1d83f743ce806bd1ea2256fc7f 2024-11-17T15:39:14,359 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/C/7d7ea2aef7e84bacbcbc0b4905387dd6 is 50, key is test_row_0/C:col10/1731857952688/Put/seqid=0 2024-11-17T15:39:14,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742341_1517 (size=12301) 2024-11-17T15:39:14,432 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:14,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48390 deadline: 1731858014423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:14,433 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:14,433 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:14,433 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:39:14,497 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-17T15:39:14,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. 2024-11-17T15:39:14,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. as already flushing 2024-11-17T15:39:14,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. 2024-11-17T15:39:14,497 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:39:14,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:39:14,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:39:14,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-17T15:39:14,650 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-17T15:39:14,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. 2024-11-17T15:39:14,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. as already flushing 2024-11-17T15:39:14,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. 2024-11-17T15:39:14,650 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:39:14,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:39:14,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:39:14,755 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:14,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48390 deadline: 1731858014743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:14,756 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 , the old value is region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:14,756 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:14,756 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., hostname=7a780d55532c,32955,1731857800648, seqNum=2 because the exception is null or not the one we care about 2024-11-17T15:39:14,763 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=366 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/C/7d7ea2aef7e84bacbcbc0b4905387dd6 2024-11-17T15:39:14,777 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/A/1f25571fc01440a7926d56873cd7afe8 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/A/1f25571fc01440a7926d56873cd7afe8 2024-11-17T15:39:14,780 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/A/1f25571fc01440a7926d56873cd7afe8, entries=200, sequenceid=366, filesize=14.4 K 2024-11-17T15:39:14,781 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/B/a9a58f1d83f743ce806bd1ea2256fc7f as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/B/a9a58f1d83f743ce806bd1ea2256fc7f 2024-11-17T15:39:14,784 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/B/a9a58f1d83f743ce806bd1ea2256fc7f, entries=150, sequenceid=366, filesize=12.0 K 2024-11-17T15:39:14,785 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/C/7d7ea2aef7e84bacbcbc0b4905387dd6 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/C/7d7ea2aef7e84bacbcbc0b4905387dd6 2024-11-17T15:39:14,791 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/C/7d7ea2aef7e84bacbcbc0b4905387dd6, entries=150, sequenceid=366, filesize=12.0 K 2024-11-17T15:39:14,792 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 420677de373b14e08d29085670121ff4 in 874ms, sequenceid=366, compaction requested=true 2024-11-17T15:39:14,792 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 420677de373b14e08d29085670121ff4: 2024-11-17T15:39:14,792 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 420677de373b14e08d29085670121ff4:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T15:39:14,792 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T15:39:14,793 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T15:39:14,793 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 420677de373b14e08d29085670121ff4:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T15:39:14,793 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T15:39:14,793 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 420677de373b14e08d29085670121ff4:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T15:39:14,793 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-17T15:39:14,793 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T15:39:14,796 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54800 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T15:39:14,797 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1541): 420677de373b14e08d29085670121ff4/A is initiating minor compaction (all files) 2024-11-17T15:39:14,797 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 420677de373b14e08d29085670121ff4/A in TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. 2024-11-17T15:39:14,797 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/A/b5331c1409a34124b7b76601f691cdbd, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/A/f81903ead88b4fd0a310201fb06d3beb, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/A/079950a6805746e5879bc86acc82208e, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/A/1f25571fc01440a7926d56873cd7afe8] into tmpdir=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp, totalSize=53.5 K 2024-11-17T15:39:14,797 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting b5331c1409a34124b7b76601f691cdbd, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1731857950804 2024-11-17T15:39:14,797 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49920 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T15:39:14,797 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1541): 420677de373b14e08d29085670121ff4/B is initiating minor compaction (all files) 2024-11-17T15:39:14,797 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 420677de373b14e08d29085670121ff4/B in TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. 2024-11-17T15:39:14,797 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/B/a185eb8574c443faa7e5500b5c2b3ff5, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/B/f4abf2ba24034ff4935d2b7e9b0d1510, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/B/e8267f0a4da340b084f80c35ce7fd8a1, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/B/a9a58f1d83f743ce806bd1ea2256fc7f] into tmpdir=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp, totalSize=48.8 K 2024-11-17T15:39:14,797 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting f81903ead88b4fd0a310201fb06d3beb, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1731857951269 2024-11-17T15:39:14,797 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting a185eb8574c443faa7e5500b5c2b3ff5, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1731857950804 2024-11-17T15:39:14,800 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 079950a6805746e5879bc86acc82208e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=354, earliestPutTs=1731857951424 2024-11-17T15:39:14,800 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting f4abf2ba24034ff4935d2b7e9b0d1510, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1731857951269 2024-11-17T15:39:14,800 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting e8267f0a4da340b084f80c35ce7fd8a1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=354, earliestPutTs=1731857951424 2024-11-17T15:39:14,800 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1f25571fc01440a7926d56873cd7afe8, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=366, earliestPutTs=1731857952688 2024-11-17T15:39:14,800 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting a9a58f1d83f743ce806bd1ea2256fc7f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=366, earliestPutTs=1731857952688 2024-11-17T15:39:14,803 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-17T15:39:14,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. 2024-11-17T15:39:14,803 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2902): Flushing 420677de373b14e08d29085670121ff4 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-17T15:39:14,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/A/a3eadf4ac9db48168d958b020bcf97bc is 50, key is test_row_0/A:col10/1731857954087/Put/seqid=0 2024-11-17T15:39:14,810 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 420677de373b14e08d29085670121ff4#A#compaction#441 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T15:39:14,810 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/A/662b44594da74a08b69b1ed8f4d53048 is 50, key is test_row_0/A:col10/1731857952688/Put/seqid=0 2024-11-17T15:39:14,814 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 420677de373b14e08d29085670121ff4#B#compaction#442 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T15:39:14,814 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/B/b682d411315545a4949956d028ca4cd6 is 50, key is test_row_0/B:col10/1731857952688/Put/seqid=0 2024-11-17T15:39:14,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742344_1520 (size=13153) 2024-11-17T15:39:14,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742343_1519 (size=13153) 2024-11-17T15:39:14,839 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742342_1518 (size=12301) 2024-11-17T15:39:14,839 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/A/662b44594da74a08b69b1ed8f4d53048 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/A/662b44594da74a08b69b1ed8f4d53048 2024-11-17T15:39:14,840 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/B/b682d411315545a4949956d028ca4cd6 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/B/b682d411315545a4949956d028ca4cd6 2024-11-17T15:39:14,845 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 420677de373b14e08d29085670121ff4/B of 420677de373b14e08d29085670121ff4 into b682d411315545a4949956d028ca4cd6(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T15:39:14,845 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 420677de373b14e08d29085670121ff4: 2024-11-17T15:39:14,845 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., storeName=420677de373b14e08d29085670121ff4/B, priority=12, startTime=1731857954792; duration=0sec 2024-11-17T15:39:14,845 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T15:39:14,845 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 420677de373b14e08d29085670121ff4:B 2024-11-17T15:39:14,845 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T15:39:14,846 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 420677de373b14e08d29085670121ff4/A of 420677de373b14e08d29085670121ff4 into 662b44594da74a08b69b1ed8f4d53048(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T15:39:14,846 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49920 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T15:39:14,846 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 420677de373b14e08d29085670121ff4: 2024-11-17T15:39:14,846 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., storeName=420677de373b14e08d29085670121ff4/A, priority=12, startTime=1731857954792; duration=0sec 2024-11-17T15:39:14,846 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1541): 420677de373b14e08d29085670121ff4/C is initiating minor compaction (all files) 2024-11-17T15:39:14,846 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 420677de373b14e08d29085670121ff4/C in TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. 2024-11-17T15:39:14,846 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T15:39:14,846 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 420677de373b14e08d29085670121ff4:A 2024-11-17T15:39:14,846 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/C/6a4b3958b11f4403a683feef1bf61c6f, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/C/287ba7a9bf2948ea8c020c2cac89edd5, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/C/14a8b35cc74a44e79374c243455e29c5, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/C/7d7ea2aef7e84bacbcbc0b4905387dd6] into tmpdir=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp, totalSize=48.8 K 2024-11-17T15:39:14,847 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting 6a4b3958b11f4403a683feef1bf61c6f, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1731857950804 2024-11-17T15:39:14,847 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting 287ba7a9bf2948ea8c020c2cac89edd5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1731857951269 2024-11-17T15:39:14,847 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting 14a8b35cc74a44e79374c243455e29c5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=354, earliestPutTs=1731857951424 2024-11-17T15:39:14,847 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting 7d7ea2aef7e84bacbcbc0b4905387dd6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=366, earliestPutTs=1731857952688 2024-11-17T15:39:14,854 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 420677de373b14e08d29085670121ff4#C#compaction#443 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T15:39:14,855 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/C/4f58abc5c76c4c7784dd4a3969457448 is 50, key is test_row_0/C:col10/1731857952688/Put/seqid=0 2024-11-17T15:39:14,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742345_1521 (size=13153) 2024-11-17T15:39:14,933 INFO [Thread-2001 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2001. 2024-11-17T15:39:14,933 DEBUG [Thread-2001 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T15:39:14,933 DEBUG [Thread-2001 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:39:14,933 DEBUG [Thread-2001 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:39:14,933 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T15:39:14,937 INFO [Thread-2009 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2009. 2024-11-17T15:39:14,937 DEBUG [Thread-2009 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T15:39:14,937 DEBUG [Thread-2009 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:39:14,937 DEBUG [Thread-2009 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:39:14,938 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T15:39:14,939 INFO [Thread-2003 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2003. 2024-11-17T15:39:14,939 DEBUG [Thread-2003 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T15:39:14,939 DEBUG [Thread-2003 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:39:14,939 DEBUG [Thread-2003 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:39:14,939 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T15:39:14,941 INFO [Thread-2007 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2007. 2024-11-17T15:39:14,941 DEBUG [Thread-2007 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T15:39:14,941 DEBUG [Thread-2007 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:39:14,941 DEBUG [Thread-2007 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:39:14,941 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T15:39:14,943 INFO [Thread-2005 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2005. 2024-11-17T15:39:14,943 DEBUG [Thread-2005 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T15:39:14,943 DEBUG [Thread-2005 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:39:14,943 DEBUG [Thread-2005 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:39:14,943 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T15:39:15,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-17T15:39:15,240 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=390 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/A/a3eadf4ac9db48168d958b020bcf97bc 2024-11-17T15:39:15,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/B/0a2d3b5ca63146e789883017b9eb5af4 is 50, key is test_row_0/B:col10/1731857954087/Put/seqid=0 2024-11-17T15:39:15,248 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742346_1522 (size=12301) 2024-11-17T15:39:15,261 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/C/4f58abc5c76c4c7784dd4a3969457448 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/C/4f58abc5c76c4c7784dd4a3969457448 2024-11-17T15:39:15,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(8855): Flush requested on 420677de373b14e08d29085670121ff4 2024-11-17T15:39:15,264 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. as already flushing 2024-11-17T15:39:15,264 INFO [Thread-1992 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1992. 2024-11-17T15:39:15,264 DEBUG [Thread-1992 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T15:39:15,264 DEBUG [Thread-1992 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:39:15,265 DEBUG [Thread-1992 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:39:15,265 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T15:39:15,266 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 420677de373b14e08d29085670121ff4/C of 420677de373b14e08d29085670121ff4 into 4f58abc5c76c4c7784dd4a3969457448(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T15:39:15,266 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 420677de373b14e08d29085670121ff4: 2024-11-17T15:39:15,266 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., storeName=420677de373b14e08d29085670121ff4/C, priority=12, startTime=1731857954793; duration=0sec 2024-11-17T15:39:15,266 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T15:39:15,266 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 420677de373b14e08d29085670121ff4:C 2024-11-17T15:39:15,649 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=390 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/B/0a2d3b5ca63146e789883017b9eb5af4 2024-11-17T15:39:15,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/C/85741960703e4182b2657db7f6cc03f3 is 50, key is test_row_0/C:col10/1731857954087/Put/seqid=0 2024-11-17T15:39:15,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742347_1523 (size=12301) 2024-11-17T15:39:15,661 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=390 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/C/85741960703e4182b2657db7f6cc03f3 2024-11-17T15:39:15,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/A/a3eadf4ac9db48168d958b020bcf97bc as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/A/a3eadf4ac9db48168d958b020bcf97bc 2024-11-17T15:39:15,673 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/A/a3eadf4ac9db48168d958b020bcf97bc, entries=150, sequenceid=390, filesize=12.0 K 2024-11-17T15:39:15,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/B/0a2d3b5ca63146e789883017b9eb5af4 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/B/0a2d3b5ca63146e789883017b9eb5af4 2024-11-17T15:39:15,681 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/B/0a2d3b5ca63146e789883017b9eb5af4, entries=150, sequenceid=390, filesize=12.0 K 2024-11-17T15:39:15,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/C/85741960703e4182b2657db7f6cc03f3 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/C/85741960703e4182b2657db7f6cc03f3 2024-11-17T15:39:15,689 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/C/85741960703e4182b2657db7f6cc03f3, entries=150, sequenceid=390, filesize=12.0 K 2024-11-17T15:39:15,690 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=6.71 KB/6870 for 420677de373b14e08d29085670121ff4 in 887ms, sequenceid=390, compaction requested=false 2024-11-17T15:39:15,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2603): Flush status journal for 420677de373b14e08d29085670121ff4: 2024-11-17T15:39:15,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. 2024-11-17T15:39:15,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=136 2024-11-17T15:39:15,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.HMaster(4169): Remote procedure done, pid=136 2024-11-17T15:39:15,693 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-11-17T15:39:15,693 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8050 sec 2024-11-17T15:39:15,695 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees in 1.8100 sec 2024-11-17T15:39:16,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-17T15:39:16,033 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T15:39:16,155 INFO [Thread-1990 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1990. 2024-11-17T15:39:16,155 DEBUG [Thread-1990 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T15:39:16,155 DEBUG [Thread-1990 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:39:16,155 DEBUG [Thread-1990 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:39:16,155 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T15:39:16,175 INFO [Thread-1996 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1996. 2024-11-17T15:39:16,175 DEBUG [Thread-1996 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T15:39:16,175 DEBUG [Thread-1996 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:39:16,175 DEBUG [Thread-1996 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:39:16,175 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T15:39:16,194 INFO [Thread-1994 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1994. 2024-11-17T15:39:16,194 DEBUG [Thread-1994 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T15:39:16,194 DEBUG [Thread-1994 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:39:16,194 DEBUG [Thread-1994 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:39:16,194 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T15:39:23,455 INFO [Thread-1998 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1998. 2024-11-17T15:39:23,455 DEBUG [Thread-1998 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T15:39:23,455 DEBUG [Thread-1998 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:39:23,455 DEBUG [Thread-1998 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:39:23,455 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T15:39:23,456 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-17T15:39:23,456 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 57 2024-11-17T15:39:23,456 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 142 2024-11-17T15:39:23,456 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 55 2024-11-17T15:39:23,456 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 54 2024-11-17T15:39:23,456 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 8 2024-11-17T15:39:23,456 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-17T15:39:23,456 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-17T15:39:23,456 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1305 2024-11-17T15:39:23,456 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3914 rows 2024-11-17T15:39:23,456 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1306 2024-11-17T15:39:23,456 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3913 rows 2024-11-17T15:39:23,456 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1309 2024-11-17T15:39:23,456 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3926 rows 2024-11-17T15:39:23,456 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1310 2024-11-17T15:39:23,456 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3929 rows 2024-11-17T15:39:23,456 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1306 2024-11-17T15:39:23,456 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3911 rows 2024-11-17T15:39:23,456 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-17T15:39:23,456 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-17T15:39:23,456 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:39:23,456 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:39:23,456 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T15:39:23,457 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-17T15:39:23,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] procedure2.ProcedureExecutor(1139): Stored pid=137, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-17T15:39:23,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-17T15:39:23,460 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731857963460"}]},"ts":"1731857963460"} 2024-11-17T15:39:23,461 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-17T15:39:23,461 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-17T15:39:23,462 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-17T15:39:23,463 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=420677de373b14e08d29085670121ff4, UNASSIGN}] 2024-11-17T15:39:23,464 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=420677de373b14e08d29085670121ff4, UNASSIGN 2024-11-17T15:39:23,465 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=139 updating hbase:meta row=420677de373b14e08d29085670121ff4, regionState=CLOSING, regionLocation=7a780d55532c,32955,1731857800648 2024-11-17T15:39:23,466 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=420677de373b14e08d29085670121ff4, UNASSIGN because future has completed 2024-11-17T15:39:23,466 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-17T15:39:23,466 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE, hasLock=false; CloseRegionProcedure 420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648}] 2024-11-17T15:39:23,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-17T15:39:23,618 INFO [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] handler.UnassignRegionHandler(122): Close 420677de373b14e08d29085670121ff4 2024-11-17T15:39:23,618 DEBUG [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-17T15:39:23,618 DEBUG [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1722): Closing 420677de373b14e08d29085670121ff4, disabling compactions & flushes 2024-11-17T15:39:23,618 INFO [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. 2024-11-17T15:39:23,618 DEBUG [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. 2024-11-17T15:39:23,618 DEBUG [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. after waiting 0 ms 2024-11-17T15:39:23,618 DEBUG [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. 2024-11-17T15:39:23,619 INFO [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(2902): Flushing 420677de373b14e08d29085670121ff4 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-17T15:39:23,623 DEBUG [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/A/f1c821f43f044277ae5c6d8ffc14f9e6 is 50, key is test_row_0/A:col10/1731857963453/Put/seqid=0 2024-11-17T15:39:23,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742348_1524 (size=12301) 2024-11-17T15:39:23,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-17T15:39:24,026 INFO [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=401 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/A/f1c821f43f044277ae5c6d8ffc14f9e6 2024-11-17T15:39:24,033 DEBUG [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/B/f740e5ef12a24a968444601a033320df is 50, key is test_row_0/B:col10/1731857963453/Put/seqid=0 2024-11-17T15:39:24,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742349_1525 (size=12301) 2024-11-17T15:39:24,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-17T15:39:24,438 INFO [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=401 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/B/f740e5ef12a24a968444601a033320df 2024-11-17T15:39:24,444 DEBUG [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/C/b11f053edd94471fb4ae860ac6feda0c is 50, key is test_row_0/C:col10/1731857963453/Put/seqid=0 2024-11-17T15:39:24,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742350_1526 (size=12301) 2024-11-17T15:39:24,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-17T15:39:24,847 INFO [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=401 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/C/b11f053edd94471fb4ae860ac6feda0c 2024-11-17T15:39:24,850 DEBUG [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/A/f1c821f43f044277ae5c6d8ffc14f9e6 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/A/f1c821f43f044277ae5c6d8ffc14f9e6 2024-11-17T15:39:24,853 INFO [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/A/f1c821f43f044277ae5c6d8ffc14f9e6, entries=150, sequenceid=401, filesize=12.0 K 2024-11-17T15:39:24,854 DEBUG [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/B/f740e5ef12a24a968444601a033320df as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/B/f740e5ef12a24a968444601a033320df 2024-11-17T15:39:24,856 INFO [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/B/f740e5ef12a24a968444601a033320df, entries=150, sequenceid=401, filesize=12.0 K 2024-11-17T15:39:24,857 DEBUG [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/.tmp/C/b11f053edd94471fb4ae860ac6feda0c as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/C/b11f053edd94471fb4ae860ac6feda0c 2024-11-17T15:39:24,859 INFO [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/C/b11f053edd94471fb4ae860ac6feda0c, entries=150, sequenceid=401, filesize=12.0 K 2024-11-17T15:39:24,860 INFO [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 420677de373b14e08d29085670121ff4 in 1242ms, sequenceid=401, compaction requested=true 2024-11-17T15:39:24,861 DEBUG [StoreCloser-TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/A/fac8538d881540e09b34eccbbbd560a5, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/A/9c313cd408cd44719b994aeb5213bc54, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/A/91d6f5810c9049b3862101472bea8900, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/A/bb8bbbc8baae426890539d5d7bed9c5f, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/A/e60042d919a04568be5b1728416beef3, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/A/4805b565078b4a26bd2e96d309c9fdbe, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/A/4394999c3c434c6f8b2298469d5105ae, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/A/10f43bae05674d589709bd59a71522b0, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/A/09d43038f697433782aaa7db76a91ea7, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/A/867380b104334bdcaf817efa0fd6eb23, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/A/31f2e33148a74c68ad38cdfa77f5dd2a, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/A/5fbd9a41a7914715a6a3a47d2f58389c, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/A/8827c182f10443518368b2bf73442781, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/A/52fca7712eaf44f280c61ddd400d6e96, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/A/0721721744834162a605031969826d25, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/A/bd253feab9074b2cabea2a742df66b24, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/A/63eb6d0797b746799e9924fe880b94da, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/A/b198e30eceaa4e879ad52a0c747d2a4a, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/A/be48caf0e9ea402ebcadc08889269990, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/A/64c2d140fac44f5f8829bad8dec76694, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/A/983081ef739f484ebe035b423402432e, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/A/b5331c1409a34124b7b76601f691cdbd, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/A/f81903ead88b4fd0a310201fb06d3beb, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/A/079950a6805746e5879bc86acc82208e, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/A/1f25571fc01440a7926d56873cd7afe8] to archive 2024-11-17T15:39:24,861 DEBUG [StoreCloser-TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-17T15:39:24,863 DEBUG [StoreCloser-TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/A/fac8538d881540e09b34eccbbbd560a5 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/A/fac8538d881540e09b34eccbbbd560a5 2024-11-17T15:39:24,863 DEBUG [StoreCloser-TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/A/9c313cd408cd44719b994aeb5213bc54 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/A/9c313cd408cd44719b994aeb5213bc54 2024-11-17T15:39:24,864 DEBUG [StoreCloser-TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/A/91d6f5810c9049b3862101472bea8900 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/A/91d6f5810c9049b3862101472bea8900 2024-11-17T15:39:24,865 DEBUG [StoreCloser-TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/A/bb8bbbc8baae426890539d5d7bed9c5f to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/A/bb8bbbc8baae426890539d5d7bed9c5f 2024-11-17T15:39:24,866 DEBUG [StoreCloser-TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/A/e60042d919a04568be5b1728416beef3 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/A/e60042d919a04568be5b1728416beef3 2024-11-17T15:39:24,866 DEBUG [StoreCloser-TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/A/4805b565078b4a26bd2e96d309c9fdbe to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/A/4805b565078b4a26bd2e96d309c9fdbe 2024-11-17T15:39:24,867 DEBUG [StoreCloser-TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/A/4394999c3c434c6f8b2298469d5105ae to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/A/4394999c3c434c6f8b2298469d5105ae 2024-11-17T15:39:24,868 DEBUG [StoreCloser-TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/A/10f43bae05674d589709bd59a71522b0 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/A/10f43bae05674d589709bd59a71522b0 2024-11-17T15:39:24,869 DEBUG [StoreCloser-TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/A/09d43038f697433782aaa7db76a91ea7 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/A/09d43038f697433782aaa7db76a91ea7 2024-11-17T15:39:24,869 DEBUG [StoreCloser-TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/A/867380b104334bdcaf817efa0fd6eb23 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/A/867380b104334bdcaf817efa0fd6eb23 2024-11-17T15:39:24,870 DEBUG [StoreCloser-TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/A/31f2e33148a74c68ad38cdfa77f5dd2a to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/A/31f2e33148a74c68ad38cdfa77f5dd2a 2024-11-17T15:39:24,871 DEBUG [StoreCloser-TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/A/5fbd9a41a7914715a6a3a47d2f58389c to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/A/5fbd9a41a7914715a6a3a47d2f58389c 2024-11-17T15:39:24,872 DEBUG [StoreCloser-TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/A/8827c182f10443518368b2bf73442781 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/A/8827c182f10443518368b2bf73442781 2024-11-17T15:39:24,873 DEBUG [StoreCloser-TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/A/52fca7712eaf44f280c61ddd400d6e96 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/A/52fca7712eaf44f280c61ddd400d6e96 2024-11-17T15:39:24,874 DEBUG [StoreCloser-TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/A/0721721744834162a605031969826d25 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/A/0721721744834162a605031969826d25 2024-11-17T15:39:24,875 DEBUG [StoreCloser-TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/A/bd253feab9074b2cabea2a742df66b24 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/A/bd253feab9074b2cabea2a742df66b24 2024-11-17T15:39:24,876 DEBUG [StoreCloser-TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/A/63eb6d0797b746799e9924fe880b94da to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/A/63eb6d0797b746799e9924fe880b94da 2024-11-17T15:39:24,876 DEBUG [StoreCloser-TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/A/b198e30eceaa4e879ad52a0c747d2a4a to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/A/b198e30eceaa4e879ad52a0c747d2a4a 2024-11-17T15:39:24,877 DEBUG [StoreCloser-TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/A/be48caf0e9ea402ebcadc08889269990 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/A/be48caf0e9ea402ebcadc08889269990 2024-11-17T15:39:24,877 DEBUG [StoreCloser-TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/A/64c2d140fac44f5f8829bad8dec76694 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/A/64c2d140fac44f5f8829bad8dec76694 2024-11-17T15:39:24,878 DEBUG [StoreCloser-TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/A/983081ef739f484ebe035b423402432e to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/A/983081ef739f484ebe035b423402432e 2024-11-17T15:39:24,879 DEBUG [StoreCloser-TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/A/b5331c1409a34124b7b76601f691cdbd to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/A/b5331c1409a34124b7b76601f691cdbd 2024-11-17T15:39:24,879 DEBUG [StoreCloser-TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/A/f81903ead88b4fd0a310201fb06d3beb to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/A/f81903ead88b4fd0a310201fb06d3beb 2024-11-17T15:39:24,880 DEBUG [StoreCloser-TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/A/079950a6805746e5879bc86acc82208e to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/A/079950a6805746e5879bc86acc82208e 2024-11-17T15:39:24,881 DEBUG [StoreCloser-TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/A/1f25571fc01440a7926d56873cd7afe8 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/A/1f25571fc01440a7926d56873cd7afe8 2024-11-17T15:39:24,882 DEBUG [StoreCloser-TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/B/1634d6811e2e49e7bcbccf469dc01b0f, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/B/54e7e6c4ca354a46ab46303f28a686c2, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/B/83e66da322354814add77a9da6c7c038, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/B/d029d99f5dc94fb69fb270c6feeb1f7d, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/B/e7d510adc3e24559b6ab12953fc5ffa0, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/B/0b8d8baf03fb458bb96a3e4f636b4eee, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/B/69c77f3423dc4110965c7be2515679aa, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/B/a53430876a9a43e58b166401b120d17a, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/B/4abb95086c6d46b2962c6a111ee546ad, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/B/7190ae69d3554889aa12e1235f491bb8, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/B/2b1e1c82825e4af08678b9857d72a0d2, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/B/7b8d6451dfa249f8b7f7c31ef8c9ba64, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/B/c8f4cde97c46447b84e2c92845b60c4a, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/B/1e3287eebbf24045a49d143486988d98, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/B/dd9f96e826f84a7d8486fa982a959410, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/B/9d23c82e83874c45aec686107121ab31, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/B/8f45c7be662d4b47b61acd4f67767e6e, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/B/ab9fddf0e1d64b489a716ace383c7558, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/B/b6f5e8c4cb7040f0856dc79bc819bdb1, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/B/2314b67e48464084a68b27d4cb08de46, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/B/a185eb8574c443faa7e5500b5c2b3ff5, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/B/723fc23987244c0681bd591019641aa3, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/B/f4abf2ba24034ff4935d2b7e9b0d1510, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/B/e8267f0a4da340b084f80c35ce7fd8a1, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/B/a9a58f1d83f743ce806bd1ea2256fc7f] to archive 2024-11-17T15:39:24,883 DEBUG [StoreCloser-TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-17T15:39:24,884 DEBUG [StoreCloser-TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/B/1634d6811e2e49e7bcbccf469dc01b0f to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/B/1634d6811e2e49e7bcbccf469dc01b0f 2024-11-17T15:39:24,884 DEBUG [StoreCloser-TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/B/54e7e6c4ca354a46ab46303f28a686c2 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/B/54e7e6c4ca354a46ab46303f28a686c2 2024-11-17T15:39:24,885 DEBUG [StoreCloser-TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/B/83e66da322354814add77a9da6c7c038 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/B/83e66da322354814add77a9da6c7c038 2024-11-17T15:39:24,886 DEBUG [StoreCloser-TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/B/d029d99f5dc94fb69fb270c6feeb1f7d to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/B/d029d99f5dc94fb69fb270c6feeb1f7d 2024-11-17T15:39:24,887 DEBUG [StoreCloser-TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/B/e7d510adc3e24559b6ab12953fc5ffa0 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/B/e7d510adc3e24559b6ab12953fc5ffa0 2024-11-17T15:39:24,888 DEBUG [StoreCloser-TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/B/0b8d8baf03fb458bb96a3e4f636b4eee to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/B/0b8d8baf03fb458bb96a3e4f636b4eee 2024-11-17T15:39:24,888 DEBUG [StoreCloser-TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/B/69c77f3423dc4110965c7be2515679aa to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/B/69c77f3423dc4110965c7be2515679aa 2024-11-17T15:39:24,889 DEBUG [StoreCloser-TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/B/a53430876a9a43e58b166401b120d17a to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/B/a53430876a9a43e58b166401b120d17a 2024-11-17T15:39:24,890 DEBUG [StoreCloser-TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/B/4abb95086c6d46b2962c6a111ee546ad to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/B/4abb95086c6d46b2962c6a111ee546ad 2024-11-17T15:39:24,890 DEBUG [StoreCloser-TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/B/7190ae69d3554889aa12e1235f491bb8 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/B/7190ae69d3554889aa12e1235f491bb8 2024-11-17T15:39:24,891 DEBUG [StoreCloser-TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/B/2b1e1c82825e4af08678b9857d72a0d2 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/B/2b1e1c82825e4af08678b9857d72a0d2 2024-11-17T15:39:24,891 DEBUG [StoreCloser-TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/B/7b8d6451dfa249f8b7f7c31ef8c9ba64 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/B/7b8d6451dfa249f8b7f7c31ef8c9ba64 2024-11-17T15:39:24,892 DEBUG [StoreCloser-TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/B/c8f4cde97c46447b84e2c92845b60c4a to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/B/c8f4cde97c46447b84e2c92845b60c4a 2024-11-17T15:39:24,893 DEBUG [StoreCloser-TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/B/1e3287eebbf24045a49d143486988d98 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/B/1e3287eebbf24045a49d143486988d98 2024-11-17T15:39:24,894 DEBUG [StoreCloser-TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/B/dd9f96e826f84a7d8486fa982a959410 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/B/dd9f96e826f84a7d8486fa982a959410 2024-11-17T15:39:24,894 DEBUG [StoreCloser-TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/B/9d23c82e83874c45aec686107121ab31 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/B/9d23c82e83874c45aec686107121ab31 2024-11-17T15:39:24,895 DEBUG [StoreCloser-TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/B/8f45c7be662d4b47b61acd4f67767e6e to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/B/8f45c7be662d4b47b61acd4f67767e6e 2024-11-17T15:39:24,896 DEBUG [StoreCloser-TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/B/ab9fddf0e1d64b489a716ace383c7558 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/B/ab9fddf0e1d64b489a716ace383c7558 2024-11-17T15:39:24,896 DEBUG [StoreCloser-TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/B/b6f5e8c4cb7040f0856dc79bc819bdb1 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/B/b6f5e8c4cb7040f0856dc79bc819bdb1 2024-11-17T15:39:24,897 DEBUG [StoreCloser-TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/B/2314b67e48464084a68b27d4cb08de46 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/B/2314b67e48464084a68b27d4cb08de46 2024-11-17T15:39:24,898 DEBUG [StoreCloser-TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/B/a185eb8574c443faa7e5500b5c2b3ff5 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/B/a185eb8574c443faa7e5500b5c2b3ff5 2024-11-17T15:39:24,899 DEBUG [StoreCloser-TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/B/723fc23987244c0681bd591019641aa3 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/B/723fc23987244c0681bd591019641aa3 2024-11-17T15:39:24,899 DEBUG [StoreCloser-TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/B/f4abf2ba24034ff4935d2b7e9b0d1510 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/B/f4abf2ba24034ff4935d2b7e9b0d1510 2024-11-17T15:39:24,900 DEBUG [StoreCloser-TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/B/e8267f0a4da340b084f80c35ce7fd8a1 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/B/e8267f0a4da340b084f80c35ce7fd8a1 2024-11-17T15:39:24,901 DEBUG [StoreCloser-TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/B/a9a58f1d83f743ce806bd1ea2256fc7f to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/B/a9a58f1d83f743ce806bd1ea2256fc7f 2024-11-17T15:39:24,902 DEBUG [StoreCloser-TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/C/49fd28d38a584da58dcd21d8858e73d1, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/C/ba0aafb62dc14027a9872333e1007e04, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/C/3e5a1aef52334141ac642133c0432a17, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/C/49277a44575845a98e31f8543f9639b5, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/C/528eef393c914d15a2fdd5474b1313f9, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/C/1c77f3753b7c4b099451182a0f1eca2a, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/C/fea0a1f0733448d98beb225b2135f127, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/C/05801731e76845bd81dcf316e26027c3, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/C/fc36bd9960e0461ba3b57b8389955576, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/C/7ee62f6a879e45a182ca1171bfaec364, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/C/983dee40eac14c3e99314204d759d9ee, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/C/c2b293984fc445828812b5b511b8a53a, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/C/bcc5ee4ef2cd489482d25b0178860951, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/C/ea6119db37fc4e0bba6d427518cd3e8a, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/C/26677ca8f30549be9190fce2c95fbd59, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/C/1c4c5e3c44d440ceac65d9b48b26f38a, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/C/b1686b64b37549daa2e400cb16667353, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/C/13469041a4974862a4869a04aa3c37e2, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/C/04bc1014130144aba688ba0644a0aded, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/C/ca3df9f84ef148d298c3a4faf492cb38, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/C/6a4b3958b11f4403a683feef1bf61c6f, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/C/0913d564e47b417bbb66f007ab8781fd, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/C/287ba7a9bf2948ea8c020c2cac89edd5, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/C/14a8b35cc74a44e79374c243455e29c5, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/C/7d7ea2aef7e84bacbcbc0b4905387dd6] to archive 2024-11-17T15:39:24,902 DEBUG [StoreCloser-TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-17T15:39:24,903 DEBUG [StoreCloser-TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/C/49fd28d38a584da58dcd21d8858e73d1 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/C/49fd28d38a584da58dcd21d8858e73d1 2024-11-17T15:39:24,904 DEBUG [StoreCloser-TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/C/ba0aafb62dc14027a9872333e1007e04 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/C/ba0aafb62dc14027a9872333e1007e04 2024-11-17T15:39:24,905 DEBUG [StoreCloser-TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/C/3e5a1aef52334141ac642133c0432a17 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/C/3e5a1aef52334141ac642133c0432a17 2024-11-17T15:39:24,906 DEBUG [StoreCloser-TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/C/49277a44575845a98e31f8543f9639b5 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/C/49277a44575845a98e31f8543f9639b5 2024-11-17T15:39:24,907 DEBUG [StoreCloser-TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/C/528eef393c914d15a2fdd5474b1313f9 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/C/528eef393c914d15a2fdd5474b1313f9 2024-11-17T15:39:24,908 DEBUG [StoreCloser-TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/C/1c77f3753b7c4b099451182a0f1eca2a to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/C/1c77f3753b7c4b099451182a0f1eca2a 2024-11-17T15:39:24,909 DEBUG [StoreCloser-TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/C/fea0a1f0733448d98beb225b2135f127 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/C/fea0a1f0733448d98beb225b2135f127 2024-11-17T15:39:24,909 DEBUG [StoreCloser-TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/C/05801731e76845bd81dcf316e26027c3 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/C/05801731e76845bd81dcf316e26027c3 2024-11-17T15:39:24,910 DEBUG [StoreCloser-TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/C/fc36bd9960e0461ba3b57b8389955576 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/C/fc36bd9960e0461ba3b57b8389955576 2024-11-17T15:39:24,911 DEBUG [StoreCloser-TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/C/7ee62f6a879e45a182ca1171bfaec364 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/C/7ee62f6a879e45a182ca1171bfaec364 2024-11-17T15:39:24,912 DEBUG [StoreCloser-TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/C/983dee40eac14c3e99314204d759d9ee to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/C/983dee40eac14c3e99314204d759d9ee 2024-11-17T15:39:24,913 DEBUG [StoreCloser-TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/C/c2b293984fc445828812b5b511b8a53a to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/C/c2b293984fc445828812b5b511b8a53a 2024-11-17T15:39:24,914 DEBUG [StoreCloser-TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/C/bcc5ee4ef2cd489482d25b0178860951 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/C/bcc5ee4ef2cd489482d25b0178860951 2024-11-17T15:39:24,915 DEBUG [StoreCloser-TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/C/ea6119db37fc4e0bba6d427518cd3e8a to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/C/ea6119db37fc4e0bba6d427518cd3e8a 2024-11-17T15:39:24,916 DEBUG [StoreCloser-TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/C/26677ca8f30549be9190fce2c95fbd59 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/C/26677ca8f30549be9190fce2c95fbd59 2024-11-17T15:39:24,918 DEBUG [StoreCloser-TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/C/1c4c5e3c44d440ceac65d9b48b26f38a to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/C/1c4c5e3c44d440ceac65d9b48b26f38a 2024-11-17T15:39:24,918 DEBUG [StoreCloser-TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/C/b1686b64b37549daa2e400cb16667353 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/C/b1686b64b37549daa2e400cb16667353 2024-11-17T15:39:24,920 DEBUG [StoreCloser-TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/C/13469041a4974862a4869a04aa3c37e2 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/C/13469041a4974862a4869a04aa3c37e2 2024-11-17T15:39:24,921 DEBUG [StoreCloser-TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/C/04bc1014130144aba688ba0644a0aded to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/C/04bc1014130144aba688ba0644a0aded 2024-11-17T15:39:24,922 DEBUG [StoreCloser-TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/C/ca3df9f84ef148d298c3a4faf492cb38 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/C/ca3df9f84ef148d298c3a4faf492cb38 2024-11-17T15:39:24,923 DEBUG [StoreCloser-TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/C/6a4b3958b11f4403a683feef1bf61c6f to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/C/6a4b3958b11f4403a683feef1bf61c6f 2024-11-17T15:39:24,924 DEBUG [StoreCloser-TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/C/0913d564e47b417bbb66f007ab8781fd to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/C/0913d564e47b417bbb66f007ab8781fd 2024-11-17T15:39:24,925 DEBUG [StoreCloser-TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/C/287ba7a9bf2948ea8c020c2cac89edd5 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/C/287ba7a9bf2948ea8c020c2cac89edd5 2024-11-17T15:39:24,930 DEBUG [StoreCloser-TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/C/14a8b35cc74a44e79374c243455e29c5 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/C/14a8b35cc74a44e79374c243455e29c5 2024-11-17T15:39:24,931 DEBUG [StoreCloser-TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/C/7d7ea2aef7e84bacbcbc0b4905387dd6 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/C/7d7ea2aef7e84bacbcbc0b4905387dd6 2024-11-17T15:39:24,937 DEBUG [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/recovered.edits/404.seqid, newMaxSeqId=404, maxSeqId=1 2024-11-17T15:39:24,937 INFO [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. 2024-11-17T15:39:24,937 DEBUG [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1676): Region close journal for 420677de373b14e08d29085670121ff4: Waiting for close lock at 1731857963618Running coprocessor pre-close hooks at 1731857963618Disabling compacts and flushes for region at 1731857963618Disabling writes for close at 1731857963618Obtaining lock to block concurrent updates at 1731857963619 (+1 ms)Preparing flush snapshotting stores in 420677de373b14e08d29085670121ff4 at 1731857963619Finished memstore snapshotting TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1731857963619Flushing stores of TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4. at 1731857963619Flushing 420677de373b14e08d29085670121ff4/A: creating writer at 1731857963619Flushing 420677de373b14e08d29085670121ff4/A: appending metadata at 1731857963622 (+3 ms)Flushing 420677de373b14e08d29085670121ff4/A: closing flushed file at 1731857963622Flushing 420677de373b14e08d29085670121ff4/B: creating writer at 1731857964030 (+408 ms)Flushing 420677de373b14e08d29085670121ff4/B: appending metadata at 1731857964033 (+3 ms)Flushing 420677de373b14e08d29085670121ff4/B: closing flushed file at 1731857964033Flushing 420677de373b14e08d29085670121ff4/C: creating writer at 1731857964441 (+408 ms)Flushing 420677de373b14e08d29085670121ff4/C: appending metadata at 1731857964443 (+2 ms)Flushing 420677de373b14e08d29085670121ff4/C: closing flushed file at 1731857964443Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3f6b06ff: reopening flushed file at 1731857964850 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6c8ebc55: reopening flushed file at 1731857964853 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@dce05a9: reopening flushed file at 1731857964856 (+3 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 420677de373b14e08d29085670121ff4 in 1242ms, sequenceid=401, compaction requested=true at 1731857964860 (+4 ms)Writing region close event to WAL at 1731857964934 (+74 ms)Running coprocessor post-close hooks at 1731857964937 (+3 ms)Closed at 1731857964937 2024-11-17T15:39:24,940 INFO [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] handler.UnassignRegionHandler(157): Closed 420677de373b14e08d29085670121ff4 2024-11-17T15:39:24,941 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=139 updating hbase:meta row=420677de373b14e08d29085670121ff4, regionState=CLOSED 2024-11-17T15:39:24,943 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=140, ppid=139, state=RUNNABLE, hasLock=false; CloseRegionProcedure 420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 because future has completed 2024-11-17T15:39:24,945 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=140, resume processing ppid=139 2024-11-17T15:39:24,945 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, ppid=139, state=SUCCESS, hasLock=false; CloseRegionProcedure 420677de373b14e08d29085670121ff4, server=7a780d55532c,32955,1731857800648 in 1.4770 sec 2024-11-17T15:39:24,946 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=139, resume processing ppid=138 2024-11-17T15:39:24,946 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, ppid=138, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=420677de373b14e08d29085670121ff4, UNASSIGN in 1.4820 sec 2024-11-17T15:39:24,948 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137 2024-11-17T15:39:24,948 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.4850 sec 2024-11-17T15:39:24,949 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731857964949"}]},"ts":"1731857964949"} 2024-11-17T15:39:24,950 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-17T15:39:24,950 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-17T15:39:24,951 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.4930 sec 2024-11-17T15:39:25,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-17T15:39:25,604 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-17T15:39:25,605 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-17T15:39:25,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] procedure2.ProcedureExecutor(1139): Stored pid=141, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T15:39:25,606 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=141, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T15:39:25,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-17T15:39:25,607 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=141, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T15:39:25,609 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4 2024-11-17T15:39:25,611 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/A, FileablePath, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/B, FileablePath, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/C, FileablePath, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/recovered.edits] 2024-11-17T15:39:25,613 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/A/662b44594da74a08b69b1ed8f4d53048 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/A/662b44594da74a08b69b1ed8f4d53048 2024-11-17T15:39:25,614 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/A/a3eadf4ac9db48168d958b020bcf97bc to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/A/a3eadf4ac9db48168d958b020bcf97bc 2024-11-17T15:39:25,615 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/A/f1c821f43f044277ae5c6d8ffc14f9e6 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/A/f1c821f43f044277ae5c6d8ffc14f9e6 2024-11-17T15:39:25,616 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/B/0a2d3b5ca63146e789883017b9eb5af4 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/B/0a2d3b5ca63146e789883017b9eb5af4 2024-11-17T15:39:25,617 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/B/b682d411315545a4949956d028ca4cd6 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/B/b682d411315545a4949956d028ca4cd6 2024-11-17T15:39:25,618 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/B/f740e5ef12a24a968444601a033320df to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/B/f740e5ef12a24a968444601a033320df 2024-11-17T15:39:25,620 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/C/4f58abc5c76c4c7784dd4a3969457448 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/C/4f58abc5c76c4c7784dd4a3969457448 2024-11-17T15:39:25,621 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/C/85741960703e4182b2657db7f6cc03f3 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/C/85741960703e4182b2657db7f6cc03f3 2024-11-17T15:39:25,622 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/C/b11f053edd94471fb4ae860ac6feda0c to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/C/b11f053edd94471fb4ae860ac6feda0c 2024-11-17T15:39:25,624 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/recovered.edits/404.seqid to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4/recovered.edits/404.seqid 2024-11-17T15:39:25,624 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/420677de373b14e08d29085670121ff4 2024-11-17T15:39:25,624 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-17T15:39:25,626 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=141, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T15:39:25,628 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-17T15:39:25,629 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-17T15:39:25,630 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=141, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T15:39:25,630 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-17T15:39:25,630 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731857965630"}]},"ts":"9223372036854775807"} 2024-11-17T15:39:25,632 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-17T15:39:25,632 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 420677de373b14e08d29085670121ff4, NAME => 'TestAcidGuarantees,,1731857932741.420677de373b14e08d29085670121ff4.', STARTKEY => '', ENDKEY => ''}] 2024-11-17T15:39:25,632 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-17T15:39:25,632 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731857965632"}]},"ts":"9223372036854775807"} 2024-11-17T15:39:25,634 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-17T15:39:25,634 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=141, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T15:39:25,635 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 29 msec 2024-11-17T15:39:25,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-17T15:39:25,714 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-17T15:39:25,714 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-17T15:39:25,723 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testScanAtomicity Thread=237 (was 241), OpenFileDescriptor=443 (was 455), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=389 (was 404), ProcessCount=11 (was 11), AvailableMemoryMB=2744 (was 2902) 2024-11-17T15:39:25,732 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobGetAtomicity Thread=237, OpenFileDescriptor=443, MaxFileDescriptor=1048576, SystemLoadAverage=389, ProcessCount=11, AvailableMemoryMB=2744 2024-11-17T15:39:25,733 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-17T15:39:25,733 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-17T15:39:25,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] procedure2.ProcedureExecutor(1139): Stored pid=142, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-17T15:39:25,735 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-17T15:39:25,735 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:25,735 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 142 2024-11-17T15:39:25,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-17T15:39:25,736 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-17T15:39:25,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742351_1527 (size=959) 2024-11-17T15:39:25,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-17T15:39:26,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-17T15:39:26,142 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => a682ec5138f7084a1f2f9cd3afa94497, NAME => 'TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206 2024-11-17T15:39:26,147 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742352_1528 (size=53) 2024-11-17T15:39:26,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-17T15:39:26,549 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T15:39:26,549 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing a682ec5138f7084a1f2f9cd3afa94497, disabling compactions & flushes 2024-11-17T15:39:26,549 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497. 2024-11-17T15:39:26,549 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497. 2024-11-17T15:39:26,549 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497. after waiting 0 ms 2024-11-17T15:39:26,549 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497. 2024-11-17T15:39:26,549 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497. 2024-11-17T15:39:26,549 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for a682ec5138f7084a1f2f9cd3afa94497: Waiting for close lock at 1731857966549Disabling compacts and flushes for region at 1731857966549Disabling writes for close at 1731857966549Writing region close event to WAL at 1731857966549Closed at 1731857966549 2024-11-17T15:39:26,550 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-17T15:39:26,550 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731857966550"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731857966550"}]},"ts":"1731857966550"} 2024-11-17T15:39:26,552 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-17T15:39:26,552 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-17T15:39:26,552 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731857966552"}]},"ts":"1731857966552"} 2024-11-17T15:39:26,553 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-17T15:39:26,554 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a682ec5138f7084a1f2f9cd3afa94497, ASSIGN}] 2024-11-17T15:39:26,555 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a682ec5138f7084a1f2f9cd3afa94497, ASSIGN 2024-11-17T15:39:26,555 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a682ec5138f7084a1f2f9cd3afa94497, ASSIGN; state=OFFLINE, location=7a780d55532c,32955,1731857800648; forceNewPlan=false, retain=false 2024-11-17T15:39:26,706 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=a682ec5138f7084a1f2f9cd3afa94497, regionState=OPENING, regionLocation=7a780d55532c,32955,1731857800648 2024-11-17T15:39:26,707 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a682ec5138f7084a1f2f9cd3afa94497, ASSIGN because future has completed 2024-11-17T15:39:26,708 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=144, ppid=143, state=RUNNABLE, hasLock=false; OpenRegionProcedure a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648}] 2024-11-17T15:39:26,862 INFO [RS_OPEN_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497. 2024-11-17T15:39:26,862 DEBUG [RS_OPEN_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(7752): Opening region: {ENCODED => a682ec5138f7084a1f2f9cd3afa94497, NAME => 'TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497.', STARTKEY => '', ENDKEY => ''} 2024-11-17T15:39:26,863 DEBUG [RS_OPEN_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees a682ec5138f7084a1f2f9cd3afa94497 2024-11-17T15:39:26,863 DEBUG [RS_OPEN_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T15:39:26,863 DEBUG [RS_OPEN_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(7794): checking encryption for a682ec5138f7084a1f2f9cd3afa94497 2024-11-17T15:39:26,863 DEBUG [RS_OPEN_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(7797): checking classloading for a682ec5138f7084a1f2f9cd3afa94497 2024-11-17T15:39:26,864 INFO [StoreOpener-a682ec5138f7084a1f2f9cd3afa94497-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region a682ec5138f7084a1f2f9cd3afa94497 2024-11-17T15:39:26,865 INFO [StoreOpener-a682ec5138f7084a1f2f9cd3afa94497-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a682ec5138f7084a1f2f9cd3afa94497 columnFamilyName A 2024-11-17T15:39:26,865 DEBUG [StoreOpener-a682ec5138f7084a1f2f9cd3afa94497-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:26,865 INFO [StoreOpener-a682ec5138f7084a1f2f9cd3afa94497-1 {}] regionserver.HStore(327): Store=a682ec5138f7084a1f2f9cd3afa94497/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T15:39:26,865 INFO [StoreOpener-a682ec5138f7084a1f2f9cd3afa94497-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region a682ec5138f7084a1f2f9cd3afa94497 2024-11-17T15:39:26,866 INFO [StoreOpener-a682ec5138f7084a1f2f9cd3afa94497-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a682ec5138f7084a1f2f9cd3afa94497 columnFamilyName B 2024-11-17T15:39:26,866 DEBUG [StoreOpener-a682ec5138f7084a1f2f9cd3afa94497-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:26,867 INFO [StoreOpener-a682ec5138f7084a1f2f9cd3afa94497-1 {}] regionserver.HStore(327): Store=a682ec5138f7084a1f2f9cd3afa94497/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T15:39:26,867 INFO [StoreOpener-a682ec5138f7084a1f2f9cd3afa94497-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region a682ec5138f7084a1f2f9cd3afa94497 2024-11-17T15:39:26,868 INFO [StoreOpener-a682ec5138f7084a1f2f9cd3afa94497-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a682ec5138f7084a1f2f9cd3afa94497 columnFamilyName C 2024-11-17T15:39:26,868 DEBUG [StoreOpener-a682ec5138f7084a1f2f9cd3afa94497-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:26,868 INFO [StoreOpener-a682ec5138f7084a1f2f9cd3afa94497-1 {}] regionserver.HStore(327): Store=a682ec5138f7084a1f2f9cd3afa94497/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T15:39:26,868 DEBUG [RS_OPEN_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1038): replaying wal for a682ec5138f7084a1f2f9cd3afa94497 2024-11-17T15:39:26,869 DEBUG [RS_OPEN_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497 2024-11-17T15:39:26,869 DEBUG [RS_OPEN_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497 2024-11-17T15:39:26,870 DEBUG [RS_OPEN_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1048): stopping wal replay for a682ec5138f7084a1f2f9cd3afa94497 2024-11-17T15:39:26,870 DEBUG [RS_OPEN_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1060): Cleaning up temporary data for a682ec5138f7084a1f2f9cd3afa94497 2024-11-17T15:39:26,870 DEBUG [RS_OPEN_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-17T15:39:26,871 DEBUG [RS_OPEN_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1093): writing seq id for a682ec5138f7084a1f2f9cd3afa94497 2024-11-17T15:39:26,873 DEBUG [RS_OPEN_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-17T15:39:26,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-17T15:39:26,873 INFO [RS_OPEN_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1114): Opened a682ec5138f7084a1f2f9cd3afa94497; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69629420, jitterRate=0.03755921125411987}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-17T15:39:26,873 DEBUG [RS_OPEN_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a682ec5138f7084a1f2f9cd3afa94497 2024-11-17T15:39:26,873 DEBUG [RS_OPEN_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1006): Region open journal for a682ec5138f7084a1f2f9cd3afa94497: Running coprocessor pre-open hook at 1731857966863Writing region info on filesystem at 1731857966863Initializing all the Stores at 1731857966863Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731857966863Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731857966864 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731857966864Cleaning up temporary data from old regions at 1731857966870 (+6 ms)Running coprocessor post-open hooks at 1731857966873 (+3 ms)Region opened successfully at 1731857966873 2024-11-17T15:39:26,874 INFO [RS_OPEN_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., pid=144, masterSystemTime=1731857966860 2024-11-17T15:39:26,875 DEBUG [RS_OPEN_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497. 2024-11-17T15:39:26,875 INFO [RS_OPEN_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497. 2024-11-17T15:39:26,876 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=a682ec5138f7084a1f2f9cd3afa94497, regionState=OPEN, openSeqNum=2, regionLocation=7a780d55532c,32955,1731857800648 2024-11-17T15:39:26,877 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=144, ppid=143, state=RUNNABLE, hasLock=false; OpenRegionProcedure a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 because future has completed 2024-11-17T15:39:26,879 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=144, resume processing ppid=143 2024-11-17T15:39:26,879 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, ppid=143, state=SUCCESS, hasLock=false; OpenRegionProcedure a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 in 170 msec 2024-11-17T15:39:26,880 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=143, resume processing ppid=142 2024-11-17T15:39:26,881 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, ppid=142, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a682ec5138f7084a1f2f9cd3afa94497, ASSIGN in 325 msec 2024-11-17T15:39:26,881 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-17T15:39:26,882 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731857966881"}]},"ts":"1731857966881"} 2024-11-17T15:39:26,883 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-17T15:39:26,883 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-17T15:39:26,884 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1500 sec 2024-11-17T15:39:27,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-17T15:39:27,883 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-17T15:39:27,884 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@348042de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:39:27,884 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7a780d55532c,44223,-1 for getting cluster id 2024-11-17T15:39:27,884 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T15:39:27,885 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e01f4cd2-8971-4a73-861a-2ca0deb4f76e' 2024-11-17T15:39:27,885 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T15:39:27,885 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e01f4cd2-8971-4a73-861a-2ca0deb4f76e" 2024-11-17T15:39:27,886 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54ebb205, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:39:27,886 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7a780d55532c,44223,-1] 2024-11-17T15:39:27,886 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T15:39:27,886 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:39:27,887 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49484, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T15:39:27,887 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28bf16a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:39:27,888 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T15:39:27,888 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7a780d55532c,32955,1731857800648, seqNum=-1] 2024-11-17T15:39:27,889 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T15:39:27,889 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57276, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T15:39:27,890 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-17T15:39:27,891 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is 7a780d55532c,44223,1731857799978 2024-11-17T15:39:27,891 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@6b670fd8 2024-11-17T15:39:27,891 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-17T15:39:27,891 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49492, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-17T15:39:27,893 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-17T15:39:27,893 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-17T15:39:27,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] procedure2.ProcedureExecutor(1139): Stored pid=145, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-17T15:39:27,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742353_1529 (size=995) 2024-11-17T15:39:28,303 DEBUG [PEWorker-4 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-17T15:39:28,303 INFO [PEWorker-4 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-17T15:39:28,305 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=146, ppid=145, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-17T15:39:28,307 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a682ec5138f7084a1f2f9cd3afa94497, REOPEN/MOVE}] 2024-11-17T15:39:28,308 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a682ec5138f7084a1f2f9cd3afa94497, REOPEN/MOVE 2024-11-17T15:39:28,309 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=a682ec5138f7084a1f2f9cd3afa94497, regionState=CLOSING, regionLocation=7a780d55532c,32955,1731857800648 2024-11-17T15:39:28,310 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a682ec5138f7084a1f2f9cd3afa94497, REOPEN/MOVE because future has completed 2024-11-17T15:39:28,310 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-17T15:39:28,310 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=148, ppid=147, state=RUNNABLE, hasLock=false; CloseRegionProcedure a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648}] 2024-11-17T15:39:28,462 INFO [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] handler.UnassignRegionHandler(122): Close a682ec5138f7084a1f2f9cd3afa94497 2024-11-17T15:39:28,462 DEBUG [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-17T15:39:28,462 DEBUG [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1722): Closing a682ec5138f7084a1f2f9cd3afa94497, disabling compactions & flushes 2024-11-17T15:39:28,462 INFO [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497. 2024-11-17T15:39:28,462 DEBUG [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497. 2024-11-17T15:39:28,462 DEBUG [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497. after waiting 0 ms 2024-11-17T15:39:28,462 DEBUG [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497. 2024-11-17T15:39:28,466 DEBUG [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-17T15:39:28,466 INFO [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497. 2024-11-17T15:39:28,466 DEBUG [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1676): Region close journal for a682ec5138f7084a1f2f9cd3afa94497: Waiting for close lock at 1731857968462Running coprocessor pre-close hooks at 1731857968462Disabling compacts and flushes for region at 1731857968462Disabling writes for close at 1731857968462Writing region close event to WAL at 1731857968463 (+1 ms)Running coprocessor post-close hooks at 1731857968466 (+3 ms)Closed at 1731857968466 2024-11-17T15:39:28,466 WARN [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegionServer(3299): Not adding moved region record: a682ec5138f7084a1f2f9cd3afa94497 to self. 2024-11-17T15:39:28,468 INFO [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] handler.UnassignRegionHandler(157): Closed a682ec5138f7084a1f2f9cd3afa94497 2024-11-17T15:39:28,468 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=a682ec5138f7084a1f2f9cd3afa94497, regionState=CLOSED 2024-11-17T15:39:28,470 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=148, ppid=147, state=RUNNABLE, hasLock=false; CloseRegionProcedure a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 because future has completed 2024-11-17T15:39:28,472 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=148, resume processing ppid=147 2024-11-17T15:39:28,472 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, ppid=147, state=SUCCESS, hasLock=false; CloseRegionProcedure a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 in 161 msec 2024-11-17T15:39:28,472 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a682ec5138f7084a1f2f9cd3afa94497, REOPEN/MOVE; state=CLOSED, location=7a780d55532c,32955,1731857800648; forceNewPlan=false, retain=true 2024-11-17T15:39:28,623 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=a682ec5138f7084a1f2f9cd3afa94497, regionState=OPENING, regionLocation=7a780d55532c,32955,1731857800648 2024-11-17T15:39:28,625 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a682ec5138f7084a1f2f9cd3afa94497, REOPEN/MOVE because future has completed 2024-11-17T15:39:28,625 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=149, ppid=147, state=RUNNABLE, hasLock=false; OpenRegionProcedure a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648}] 2024-11-17T15:39:28,779 INFO [RS_OPEN_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497. 2024-11-17T15:39:28,779 DEBUG [RS_OPEN_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(7752): Opening region: {ENCODED => a682ec5138f7084a1f2f9cd3afa94497, NAME => 'TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497.', STARTKEY => '', ENDKEY => ''} 2024-11-17T15:39:28,780 DEBUG [RS_OPEN_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees a682ec5138f7084a1f2f9cd3afa94497 2024-11-17T15:39:28,780 DEBUG [RS_OPEN_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T15:39:28,780 DEBUG [RS_OPEN_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(7794): checking encryption for a682ec5138f7084a1f2f9cd3afa94497 2024-11-17T15:39:28,780 DEBUG [RS_OPEN_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(7797): checking classloading for a682ec5138f7084a1f2f9cd3afa94497 2024-11-17T15:39:28,781 INFO [StoreOpener-a682ec5138f7084a1f2f9cd3afa94497-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region a682ec5138f7084a1f2f9cd3afa94497 2024-11-17T15:39:28,781 INFO [StoreOpener-a682ec5138f7084a1f2f9cd3afa94497-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a682ec5138f7084a1f2f9cd3afa94497 columnFamilyName A 2024-11-17T15:39:28,782 DEBUG [StoreOpener-a682ec5138f7084a1f2f9cd3afa94497-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:28,782 INFO [StoreOpener-a682ec5138f7084a1f2f9cd3afa94497-1 {}] regionserver.HStore(327): Store=a682ec5138f7084a1f2f9cd3afa94497/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T15:39:28,783 INFO [StoreOpener-a682ec5138f7084a1f2f9cd3afa94497-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region a682ec5138f7084a1f2f9cd3afa94497 2024-11-17T15:39:28,783 INFO [StoreOpener-a682ec5138f7084a1f2f9cd3afa94497-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a682ec5138f7084a1f2f9cd3afa94497 columnFamilyName B 2024-11-17T15:39:28,783 DEBUG [StoreOpener-a682ec5138f7084a1f2f9cd3afa94497-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:28,784 INFO [StoreOpener-a682ec5138f7084a1f2f9cd3afa94497-1 {}] regionserver.HStore(327): Store=a682ec5138f7084a1f2f9cd3afa94497/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T15:39:28,784 INFO [StoreOpener-a682ec5138f7084a1f2f9cd3afa94497-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region a682ec5138f7084a1f2f9cd3afa94497 2024-11-17T15:39:28,784 INFO [StoreOpener-a682ec5138f7084a1f2f9cd3afa94497-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a682ec5138f7084a1f2f9cd3afa94497 columnFamilyName C 2024-11-17T15:39:28,784 DEBUG [StoreOpener-a682ec5138f7084a1f2f9cd3afa94497-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:28,784 INFO [StoreOpener-a682ec5138f7084a1f2f9cd3afa94497-1 {}] regionserver.HStore(327): Store=a682ec5138f7084a1f2f9cd3afa94497/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T15:39:28,785 DEBUG [RS_OPEN_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1038): replaying wal for a682ec5138f7084a1f2f9cd3afa94497 2024-11-17T15:39:28,785 DEBUG [RS_OPEN_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497 2024-11-17T15:39:28,786 DEBUG [RS_OPEN_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497 2024-11-17T15:39:28,786 DEBUG [RS_OPEN_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1048): stopping wal replay for a682ec5138f7084a1f2f9cd3afa94497 2024-11-17T15:39:28,786 DEBUG [RS_OPEN_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1060): Cleaning up temporary data for a682ec5138f7084a1f2f9cd3afa94497 2024-11-17T15:39:28,787 DEBUG [RS_OPEN_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-17T15:39:28,787 DEBUG [RS_OPEN_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1093): writing seq id for a682ec5138f7084a1f2f9cd3afa94497 2024-11-17T15:39:28,788 INFO [RS_OPEN_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1114): Opened a682ec5138f7084a1f2f9cd3afa94497; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66945200, jitterRate=-0.002438783645629883}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-17T15:39:28,788 DEBUG [RS_OPEN_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a682ec5138f7084a1f2f9cd3afa94497 2024-11-17T15:39:28,789 DEBUG [RS_OPEN_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1006): Region open journal for a682ec5138f7084a1f2f9cd3afa94497: Running coprocessor pre-open hook at 1731857968780Writing region info on filesystem at 1731857968780Initializing all the Stores at 1731857968780Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731857968780Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731857968781 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731857968781Cleaning up temporary data from old regions at 1731857968786 (+5 ms)Running coprocessor post-open hooks at 1731857968788 (+2 ms)Region opened successfully at 1731857968789 (+1 ms) 2024-11-17T15:39:28,789 INFO [RS_OPEN_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., pid=149, masterSystemTime=1731857968776 2024-11-17T15:39:28,791 DEBUG [RS_OPEN_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497. 2024-11-17T15:39:28,791 INFO [RS_OPEN_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497. 2024-11-17T15:39:28,791 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=a682ec5138f7084a1f2f9cd3afa94497, regionState=OPEN, openSeqNum=5, regionLocation=7a780d55532c,32955,1731857800648 2024-11-17T15:39:28,792 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=149, ppid=147, state=RUNNABLE, hasLock=false; OpenRegionProcedure a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 because future has completed 2024-11-17T15:39:28,794 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=149, resume processing ppid=147 2024-11-17T15:39:28,794 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, ppid=147, state=SUCCESS, hasLock=false; OpenRegionProcedure a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 in 168 msec 2024-11-17T15:39:28,796 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=147, resume processing ppid=146 2024-11-17T15:39:28,796 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, ppid=146, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a682ec5138f7084a1f2f9cd3afa94497, REOPEN/MOVE in 487 msec 2024-11-17T15:39:28,798 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=146, resume processing ppid=145 2024-11-17T15:39:28,798 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, ppid=145, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 491 msec 2024-11-17T15:39:28,800 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 905 msec 2024-11-17T15:39:28,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-17T15:39:28,801 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-17T15:39:28,801 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24560e66, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:39:28,801 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7a780d55532c,44223,-1 for getting cluster id 2024-11-17T15:39:28,801 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T15:39:28,802 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e01f4cd2-8971-4a73-861a-2ca0deb4f76e' 2024-11-17T15:39:28,802 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T15:39:28,802 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e01f4cd2-8971-4a73-861a-2ca0deb4f76e" 2024-11-17T15:39:28,802 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@18f4c51b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:39:28,802 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7a780d55532c,44223,-1] 2024-11-17T15:39:28,803 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T15:39:28,803 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:39:28,803 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49518, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T15:39:28,804 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@17b9c0e4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:39:28,804 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e83be8c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:39:28,804 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7a780d55532c,44223,-1 for getting cluster id 2024-11-17T15:39:28,804 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T15:39:28,805 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e01f4cd2-8971-4a73-861a-2ca0deb4f76e' 2024-11-17T15:39:28,805 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T15:39:28,805 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e01f4cd2-8971-4a73-861a-2ca0deb4f76e" 2024-11-17T15:39:28,805 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d59e82e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:39:28,805 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7a780d55532c,44223,-1] 2024-11-17T15:39:28,806 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T15:39:28,806 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:39:28,806 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49542, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T15:39:28,807 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@328ae9a6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:39:28,807 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@556fc022, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:39:28,807 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7a780d55532c,44223,-1 for getting cluster id 2024-11-17T15:39:28,807 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T15:39:28,808 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e01f4cd2-8971-4a73-861a-2ca0deb4f76e' 2024-11-17T15:39:28,808 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T15:39:28,808 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e01f4cd2-8971-4a73-861a-2ca0deb4f76e" 2024-11-17T15:39:28,808 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1abf54ec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:39:28,808 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7a780d55532c,44223,-1] 2024-11-17T15:39:28,808 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T15:39:28,809 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:39:28,809 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49564, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T15:39:28,810 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26bc159a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:39:28,810 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77590f25, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:39:28,810 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7a780d55532c,44223,-1 for getting cluster id 2024-11-17T15:39:28,810 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T15:39:28,811 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e01f4cd2-8971-4a73-861a-2ca0deb4f76e' 2024-11-17T15:39:28,811 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T15:39:28,811 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e01f4cd2-8971-4a73-861a-2ca0deb4f76e" 2024-11-17T15:39:28,811 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c2e60c9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:39:28,811 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7a780d55532c,44223,-1] 2024-11-17T15:39:28,811 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T15:39:28,811 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:39:28,812 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49584, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T15:39:28,812 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13823699, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:39:28,813 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@99b73f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:39:28,813 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7a780d55532c,44223,-1 for getting cluster id 2024-11-17T15:39:28,813 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T15:39:28,813 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e01f4cd2-8971-4a73-861a-2ca0deb4f76e' 2024-11-17T15:39:28,813 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T15:39:28,813 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e01f4cd2-8971-4a73-861a-2ca0deb4f76e" 2024-11-17T15:39:28,814 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@dac1e2d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:39:28,814 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7a780d55532c,44223,-1] 2024-11-17T15:39:28,814 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T15:39:28,814 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:39:28,814 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49606, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T15:39:28,815 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1451792, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:39:28,815 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6deae39b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:39:28,815 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7a780d55532c,44223,-1 for getting cluster id 2024-11-17T15:39:28,816 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T15:39:28,816 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e01f4cd2-8971-4a73-861a-2ca0deb4f76e' 2024-11-17T15:39:28,816 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T15:39:28,816 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e01f4cd2-8971-4a73-861a-2ca0deb4f76e" 2024-11-17T15:39:28,816 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@342ee0b1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:39:28,816 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7a780d55532c,44223,-1] 2024-11-17T15:39:28,817 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T15:39:28,817 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:39:28,817 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49630, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T15:39:28,818 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50a91c4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:39:28,818 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4cb7cff8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:39:28,818 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7a780d55532c,44223,-1 for getting cluster id 2024-11-17T15:39:28,818 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T15:39:28,819 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e01f4cd2-8971-4a73-861a-2ca0deb4f76e' 2024-11-17T15:39:28,819 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T15:39:28,819 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e01f4cd2-8971-4a73-861a-2ca0deb4f76e" 2024-11-17T15:39:28,819 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@48d0f50a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:39:28,819 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7a780d55532c,44223,-1] 2024-11-17T15:39:28,820 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T15:39:28,820 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:39:28,820 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49646, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T15:39:28,821 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75c25dcd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:39:28,822 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56ae597b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:39:28,822 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7a780d55532c,44223,-1 for getting cluster id 2024-11-17T15:39:28,822 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T15:39:28,822 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e01f4cd2-8971-4a73-861a-2ca0deb4f76e' 2024-11-17T15:39:28,822 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T15:39:28,822 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e01f4cd2-8971-4a73-861a-2ca0deb4f76e" 2024-11-17T15:39:28,823 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d2099ca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:39:28,823 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7a780d55532c,44223,-1] 2024-11-17T15:39:28,823 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T15:39:28,823 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:39:28,823 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49658, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T15:39:28,824 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@730d2ac3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:39:28,824 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39174ad3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:39:28,824 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7a780d55532c,44223,-1 for getting cluster id 2024-11-17T15:39:28,824 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T15:39:28,825 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e01f4cd2-8971-4a73-861a-2ca0deb4f76e' 2024-11-17T15:39:28,825 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T15:39:28,825 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e01f4cd2-8971-4a73-861a-2ca0deb4f76e" 2024-11-17T15:39:28,825 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d8f16ac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:39:28,825 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7a780d55532c,44223,-1] 2024-11-17T15:39:28,825 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T15:39:28,825 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:39:28,826 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49676, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T15:39:28,826 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51fd06e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:39:28,827 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e8fc71, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:39:28,827 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7a780d55532c,44223,-1 for getting cluster id 2024-11-17T15:39:28,827 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T15:39:28,827 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e01f4cd2-8971-4a73-861a-2ca0deb4f76e' 2024-11-17T15:39:28,828 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T15:39:28,828 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e01f4cd2-8971-4a73-861a-2ca0deb4f76e" 2024-11-17T15:39:28,828 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4732e519, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:39:28,828 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7a780d55532c,44223,-1] 2024-11-17T15:39:28,828 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T15:39:28,828 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:39:28,829 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49700, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T15:39:28,829 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b4bf683, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T15:39:28,829 DEBUG [Thread-2374 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T15:39:28,829 DEBUG [Thread-2376 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T15:39:28,830 DEBUG [Thread-2387 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T15:39:28,830 DEBUG [Thread-2370 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T15:39:28,830 DEBUG [Thread-2378 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T15:39:28,830 DEBUG [Thread-2372 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T15:39:28,830 DEBUG [Thread-2383 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T15:39:28,830 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7a780d55532c,32955,1731857800648, seqNum=-1] 2024-11-17T15:39:28,830 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7a780d55532c,32955,1731857800648, seqNum=-1] 2024-11-17T15:39:28,830 DEBUG [Thread-2389 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T15:39:28,831 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7a780d55532c,32955,1731857800648, seqNum=-1] 2024-11-17T15:39:28,831 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T15:39:28,831 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T15:39:28,831 DEBUG [Thread-2381 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T15:39:28,831 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T15:39:28,831 DEBUG [Thread-2385 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T15:39:28,831 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T15:39:28,831 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7a780d55532c,32955,1731857800648, seqNum=-1] 2024-11-17T15:39:28,832 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7a780d55532c,32955,1731857800648, seqNum=-1] 2024-11-17T15:39:28,832 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57294, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T15:39:28,832 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T15:39:28,832 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T15:39:28,832 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7a780d55532c,32955,1731857800648, seqNum=-1] 2024-11-17T15:39:28,832 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T15:39:28,832 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57288, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T15:39:28,832 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7a780d55532c,32955,1731857800648, seqNum=-1] 2024-11-17T15:39:28,832 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7a780d55532c,32955,1731857800648, seqNum=-1] 2024-11-17T15:39:28,833 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7a780d55532c,32955,1731857800648, seqNum=-1] 2024-11-17T15:39:28,833 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T15:39:28,833 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T15:39:28,833 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T15:39:28,833 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57322, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T15:39:28,834 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57334, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T15:39:28,834 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5] 2024-11-17T15:39:28,834 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57332, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T15:39:28,834 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57348, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T15:39:28,834 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57362, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T15:39:28,834 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5] 2024-11-17T15:39:28,835 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5] 2024-11-17T15:39:28,835 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7a780d55532c,32955,1731857800648, seqNum=-1] 2024-11-17T15:39:28,835 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5] 2024-11-17T15:39:28,835 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T15:39:28,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] procedure2.ProcedureExecutor(1139): Stored pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees 2024-11-17T15:39:28,836 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5] 2024-11-17T15:39:28,836 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57374, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T15:39:28,837 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5] 2024-11-17T15:39:28,837 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57310, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T15:39:28,837 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5] 2024-11-17T15:39:28,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-17T15:39:28,837 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57380, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T15:39:28,838 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5] 2024-11-17T15:39:28,839 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T15:39:28,839 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5] 2024-11-17T15:39:28,840 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T15:39:28,840 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=150, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T15:39:28,840 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5] 2024-11-17T15:39:28,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(8855): Flush requested on a682ec5138f7084a1f2f9cd3afa94497 2024-11-17T15:39:28,847 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T15:39:28,847 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a682ec5138f7084a1f2f9cd3afa94497 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-17T15:39:28,878 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117160e685a1a7846b482730209acdfbffb_a682ec5138f7084a1f2f9cd3afa94497 is 50, key is test_row_0/A:col10/1731857968846/Put/seqid=0 2024-11-17T15:39:28,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742354_1530 (size=14594) 2024-11-17T15:39:28,882 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:28,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57288 deadline: 1731858028877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:28,882 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:28,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57374 deadline: 1731858028877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:28,883 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:28,883 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:28,883 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:28,883 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:28,883 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:28,883 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:28,883 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:28,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57310 deadline: 1731858028878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:28,883 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:28,883 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:28,883 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:28,883 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:28,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57380 deadline: 1731858028878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:28,884 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:28,884 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:28,884 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:28,884 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:28,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57332 deadline: 1731858028879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:28,884 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:28,884 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:28,884 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:28,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-17T15:39:28,992 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-17T15:39:28,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497. 2024-11-17T15:39:28,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497. as already flushing 2024-11-17T15:39:28,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497. 2024-11-17T15:39:28,993 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => a682ec5138f7084a1f2f9cd3afa94497, NAME => 'TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:39:28,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => a682ec5138f7084a1f2f9cd3afa94497, NAME => 'TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:39:28,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a682ec5138f7084a1f2f9cd3afa94497, NAME => 'TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a682ec5138f7084a1f2f9cd3afa94497, NAME => 'TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:39:28,996 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:28,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57310 deadline: 1731858028993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:28,996 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:28,996 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:28,996 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:28,996 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:28,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57332 deadline: 1731858028993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:28,996 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:28,996 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:28,996 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:28,997 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:28,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57288 deadline: 1731858028993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:28,996 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:28,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57380 deadline: 1731858028993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:28,997 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:28,997 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:28,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57374 deadline: 1731858028993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:28,997 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:28,997 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:28,997 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:28,997 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:28,997 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:29,000 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:29,000 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:29,000 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:29,146 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-17T15:39:29,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497. 2024-11-17T15:39:29,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497. as already flushing 2024-11-17T15:39:29,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497. 2024-11-17T15:39:29,147 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => a682ec5138f7084a1f2f9cd3afa94497, NAME => 'TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:39:29,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => a682ec5138f7084a1f2f9cd3afa94497, NAME => 'TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:39:29,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a682ec5138f7084a1f2f9cd3afa94497, NAME => 'TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a682ec5138f7084a1f2f9cd3afa94497, NAME => 'TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:39:29,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-17T15:39:29,205 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:29,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57310 deadline: 1731858029203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:29,205 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:29,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57288 deadline: 1731858029203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:29,205 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:29,205 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:29,205 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:29,206 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:29,206 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:29,206 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:29,206 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:29,206 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:29,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57380 deadline: 1731858029203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:29,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57332 deadline: 1731858029203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:29,206 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:29,206 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:29,206 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:29,206 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:29,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57374 deadline: 1731858029203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:29,206 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:29,206 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:29,206 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:29,206 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:29,206 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:29,206 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:29,281 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:29,285 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117160e685a1a7846b482730209acdfbffb_a682ec5138f7084a1f2f9cd3afa94497 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117160e685a1a7846b482730209acdfbffb_a682ec5138f7084a1f2f9cd3afa94497 2024-11-17T15:39:29,286 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/A/7f803d4371504753a4d9d9f9fd4d4306, store: [table=TestAcidGuarantees family=A region=a682ec5138f7084a1f2f9cd3afa94497] 2024-11-17T15:39:29,286 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/A/7f803d4371504753a4d9d9f9fd4d4306 is 175, key is test_row_0/A:col10/1731857968846/Put/seqid=0 2024-11-17T15:39:29,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742355_1531 (size=39549) 2024-11-17T15:39:29,294 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/A/7f803d4371504753a4d9d9f9fd4d4306 2024-11-17T15:39:29,299 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-17T15:39:29,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497. 2024-11-17T15:39:29,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497. as already flushing 2024-11-17T15:39:29,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497. 2024-11-17T15:39:29,299 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => a682ec5138f7084a1f2f9cd3afa94497, NAME => 'TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:39:29,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => a682ec5138f7084a1f2f9cd3afa94497, NAME => 'TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:39:29,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a682ec5138f7084a1f2f9cd3afa94497, NAME => 'TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a682ec5138f7084a1f2f9cd3afa94497, NAME => 'TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:39:29,317 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/B/e6153b84a31b4f71a64111af4a821b0f is 50, key is test_row_0/B:col10/1731857968846/Put/seqid=0 2024-11-17T15:39:29,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742356_1532 (size=12001) 2024-11-17T15:39:29,323 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/B/e6153b84a31b4f71a64111af4a821b0f 2024-11-17T15:39:29,349 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/C/e5e5127973af48a1b821b34f6fb3c536 is 50, key is test_row_0/C:col10/1731857968846/Put/seqid=0 2024-11-17T15:39:29,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742357_1533 (size=12001) 2024-11-17T15:39:29,452 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-17T15:39:29,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497. 2024-11-17T15:39:29,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497. as already flushing 2024-11-17T15:39:29,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497. 2024-11-17T15:39:29,452 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => a682ec5138f7084a1f2f9cd3afa94497, NAME => 'TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:39:29,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => a682ec5138f7084a1f2f9cd3afa94497, NAME => 'TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:39:29,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a682ec5138f7084a1f2f9cd3afa94497, NAME => 'TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a682ec5138f7084a1f2f9cd3afa94497, NAME => 'TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:39:29,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-17T15:39:29,514 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:29,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57310 deadline: 1731858029513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:29,515 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:29,515 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:29,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57288 deadline: 1731858029513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:29,515 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:29,520 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:29,520 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:29,521 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:29,520 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:29,521 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:29,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57380 deadline: 1731858029514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:29,521 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:29,521 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:29,521 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:29,521 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:29,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57374 deadline: 1731858029514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:29,522 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:29,522 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:29,522 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:29,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57332 deadline: 1731858029514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:29,522 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:29,522 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:29,522 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:29,522 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:29,605 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-17T15:39:29,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497. 2024-11-17T15:39:29,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497. as already flushing 2024-11-17T15:39:29,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497. 2024-11-17T15:39:29,606 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => a682ec5138f7084a1f2f9cd3afa94497, NAME => 'TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:39:29,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => a682ec5138f7084a1f2f9cd3afa94497, NAME => 'TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:39:29,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a682ec5138f7084a1f2f9cd3afa94497, NAME => 'TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a682ec5138f7084a1f2f9cd3afa94497, NAME => 'TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:39:29,756 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/C/e5e5127973af48a1b821b34f6fb3c536 2024-11-17T15:39:29,760 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-17T15:39:29,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497. 2024-11-17T15:39:29,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497. as already flushing 2024-11-17T15:39:29,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497. 2024-11-17T15:39:29,760 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => a682ec5138f7084a1f2f9cd3afa94497, NAME => 'TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:39:29,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => a682ec5138f7084a1f2f9cd3afa94497, NAME => 'TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:39:29,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a682ec5138f7084a1f2f9cd3afa94497, NAME => 'TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a682ec5138f7084a1f2f9cd3afa94497, NAME => 'TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:39:29,764 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/A/7f803d4371504753a4d9d9f9fd4d4306 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/A/7f803d4371504753a4d9d9f9fd4d4306 2024-11-17T15:39:29,769 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/A/7f803d4371504753a4d9d9f9fd4d4306, entries=200, sequenceid=16, filesize=38.6 K 2024-11-17T15:39:29,771 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/B/e6153b84a31b4f71a64111af4a821b0f as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/B/e6153b84a31b4f71a64111af4a821b0f 2024-11-17T15:39:29,781 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/B/e6153b84a31b4f71a64111af4a821b0f, entries=150, sequenceid=16, filesize=11.7 K 2024-11-17T15:39:29,781 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/C/e5e5127973af48a1b821b34f6fb3c536 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/C/e5e5127973af48a1b821b34f6fb3c536 2024-11-17T15:39:29,784 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/C/e5e5127973af48a1b821b34f6fb3c536, entries=150, sequenceid=16, filesize=11.7 K 2024-11-17T15:39:29,785 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=154.31 KB/158010 for a682ec5138f7084a1f2f9cd3afa94497 in 938ms, sequenceid=16, compaction requested=false 2024-11-17T15:39:29,785 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-17T15:39:29,786 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a682ec5138f7084a1f2f9cd3afa94497: 2024-11-17T15:39:29,913 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-17T15:39:29,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497. 2024-11-17T15:39:29,913 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2902): Flushing a682ec5138f7084a1f2f9cd3afa94497 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-17T15:39:29,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411170ce47404dd9342a7a2e8483968c3d2a3_a682ec5138f7084a1f2f9cd3afa94497 is 50, key is test_row_0/A:col10/1731857968875/Put/seqid=0 2024-11-17T15:39:29,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742358_1534 (size=12154) 2024-11-17T15:39:29,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:29,938 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411170ce47404dd9342a7a2e8483968c3d2a3_a682ec5138f7084a1f2f9cd3afa94497 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411170ce47404dd9342a7a2e8483968c3d2a3_a682ec5138f7084a1f2f9cd3afa94497 2024-11-17T15:39:29,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/A/9a38772ae8b349758f03f17c9a6030b4, store: [table=TestAcidGuarantees family=A region=a682ec5138f7084a1f2f9cd3afa94497] 2024-11-17T15:39:29,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/A/9a38772ae8b349758f03f17c9a6030b4 is 175, key is test_row_0/A:col10/1731857968875/Put/seqid=0 2024-11-17T15:39:29,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742359_1535 (size=30955) 2024-11-17T15:39:29,956 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=42, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/A/9a38772ae8b349758f03f17c9a6030b4 2024-11-17T15:39:29,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/B/558ab7c6bb174c94a08e7cf77978faab is 50, key is test_row_0/B:col10/1731857968875/Put/seqid=0 2024-11-17T15:39:29,967 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742360_1536 (size=12001) 2024-11-17T15:39:29,968 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=42 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/B/558ab7c6bb174c94a08e7cf77978faab 2024-11-17T15:39:29,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/C/64c61bc118674d349cf0ea1ad693fdc4 is 50, key is test_row_0/C:col10/1731857968875/Put/seqid=0 2024-11-17T15:39:29,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-17T15:39:29,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742361_1537 (size=12001) 2024-11-17T15:39:30,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(8855): Flush requested on a682ec5138f7084a1f2f9cd3afa94497 2024-11-17T15:39:30,036 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497. as already flushing 2024-11-17T15:39:30,043 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:30,043 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:30,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57332 deadline: 1731858030037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:30,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57380 deadline: 1731858030038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:30,043 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:30,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57288 deadline: 1731858030039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:30,043 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:30,043 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:30,043 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:30,044 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:30,044 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:30,044 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:30,044 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:30,044 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:30,044 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:30,045 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:30,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57310 deadline: 1731858030042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:30,046 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:30,046 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:30,046 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:30,046 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:30,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57374 deadline: 1731858030043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:30,046 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:30,046 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:30,046 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:30,154 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:30,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57332 deadline: 1731858030153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:30,154 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:30,154 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:30,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57310 deadline: 1731858030153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:30,154 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:30,154 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:30,154 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:30,155 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:30,155 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:30,155 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:30,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57380 deadline: 1731858030153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:30,155 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:30,155 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:30,155 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:30,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57374 deadline: 1731858030153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:30,155 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:30,155 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:30,155 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:30,155 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:30,155 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:30,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57288 deadline: 1731858030153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:30,155 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:30,155 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:30,155 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:30,365 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:30,365 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:30,366 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:30,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57310 deadline: 1731858030363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:30,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57332 deadline: 1731858030363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:30,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57380 deadline: 1731858030363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:30,366 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:30,366 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:30,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57288 deadline: 1731858030363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:30,366 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:30,366 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:30,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57374 deadline: 1731858030363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:30,366 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:30,366 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:30,366 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:30,366 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:30,366 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:30,366 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:30,366 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:30,366 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:30,366 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:30,366 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:30,366 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:30,366 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:30,367 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:30,386 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=42 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/C/64c61bc118674d349cf0ea1ad693fdc4 2024-11-17T15:39:30,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/A/9a38772ae8b349758f03f17c9a6030b4 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/A/9a38772ae8b349758f03f17c9a6030b4 2024-11-17T15:39:30,393 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/A/9a38772ae8b349758f03f17c9a6030b4, entries=150, sequenceid=42, filesize=30.2 K 2024-11-17T15:39:30,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/B/558ab7c6bb174c94a08e7cf77978faab as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/B/558ab7c6bb174c94a08e7cf77978faab 2024-11-17T15:39:30,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,398 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/B/558ab7c6bb174c94a08e7cf77978faab, entries=150, sequenceid=42, filesize=11.7 K 2024-11-17T15:39:30,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/C/64c61bc118674d349cf0ea1ad693fdc4 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/C/64c61bc118674d349cf0ea1ad693fdc4 2024-11-17T15:39:30,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,403 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/C/64c61bc118674d349cf0ea1ad693fdc4, entries=150, sequenceid=42, filesize=11.7 K 2024-11-17T15:39:30,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,404 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for a682ec5138f7084a1f2f9cd3afa94497 in 491ms, sequenceid=42, compaction requested=false 2024-11-17T15:39:30,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2603): Flush status journal for a682ec5138f7084a1f2f9cd3afa94497: 2024-11-17T15:39:30,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497. 2024-11-17T15:39:30,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=151 2024-11-17T15:39:30,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.HMaster(4169): Remote procedure done, pid=151 2024-11-17T15:39:30,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,407 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=150 2024-11-17T15:39:30,407 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5650 sec 2024-11-17T15:39:30,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,410 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees in 1.5770 sec 2024-11-17T15:39:30,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,457 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-17T15:39:30,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(8855): Flush requested on a682ec5138f7084a1f2f9cd3afa94497 2024-11-17T15:39:30,677 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T15:39:30,677 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a682ec5138f7084a1f2f9cd3afa94497 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-17T15:39:30,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,687 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117bf5e6e4249894d9cba3301db3476ba0b_a682ec5138f7084a1f2f9cd3afa94497 is 50, key is test_row_0/A:col10/1731857970042/Put/seqid=0 2024-11-17T15:39:30,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742362_1538 (size=14594) 2024-11-17T15:39:30,715 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:30,719 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117bf5e6e4249894d9cba3301db3476ba0b_a682ec5138f7084a1f2f9cd3afa94497 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117bf5e6e4249894d9cba3301db3476ba0b_a682ec5138f7084a1f2f9cd3afa94497 2024-11-17T15:39:30,720 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/A/c347b5c207114d43adf46db7ab4f0d75, store: [table=TestAcidGuarantees family=A region=a682ec5138f7084a1f2f9cd3afa94497] 2024-11-17T15:39:30,720 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:30,721 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/A/c347b5c207114d43adf46db7ab4f0d75 is 175, key is test_row_0/A:col10/1731857970042/Put/seqid=0 2024-11-17T15:39:30,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57332 deadline: 1731858030712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:30,721 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:30,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57380 deadline: 1731858030713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:30,721 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:30,722 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:30,722 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:30,722 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:30,722 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:30,722 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:30,723 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:30,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57310 deadline: 1731858030719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:30,724 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:30,724 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:30,724 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:30,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57374 deadline: 1731858030719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:30,724 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:30,724 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:30,724 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:30,724 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:30,727 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:30,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57288 deadline: 1731858030721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:30,727 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:30,727 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:30,727 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:30,748 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742363_1539 (size=39549) 2024-11-17T15:39:30,748 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=54, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/A/c347b5c207114d43adf46db7ab4f0d75 2024-11-17T15:39:30,758 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/B/7b0f9b4518e5449dac6aa6827c7bb262 is 50, key is test_row_0/B:col10/1731857970042/Put/seqid=0 2024-11-17T15:39:30,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742364_1540 (size=12001) 2024-11-17T15:39:30,771 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=54 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/B/7b0f9b4518e5449dac6aa6827c7bb262 2024-11-17T15:39:30,790 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/C/b5f0237d91ac4bdb9d42b90f1f9ed19a is 50, key is test_row_0/C:col10/1731857970042/Put/seqid=0 2024-11-17T15:39:30,819 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742365_1541 (size=12001) 2024-11-17T15:39:30,819 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=54 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/C/b5f0237d91ac4bdb9d42b90f1f9ed19a 2024-11-17T15:39:30,825 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/A/c347b5c207114d43adf46db7ab4f0d75 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/A/c347b5c207114d43adf46db7ab4f0d75 2024-11-17T15:39:30,825 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:30,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57380 deadline: 1731858030823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:30,826 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:30,826 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:30,826 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:30,830 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/A/c347b5c207114d43adf46db7ab4f0d75, entries=200, sequenceid=54, filesize=38.6 K 2024-11-17T15:39:30,832 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/B/7b0f9b4518e5449dac6aa6827c7bb262 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/B/7b0f9b4518e5449dac6aa6827c7bb262 2024-11-17T15:39:30,836 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:30,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57288 deadline: 1731858030834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:30,837 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:30,837 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:30,837 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:30,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57332 deadline: 1731858030834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:30,837 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:30,837 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:30,837 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:30,837 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:30,837 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:30,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57310 deadline: 1731858030834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:30,837 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:30,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57374 deadline: 1731858030834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:30,837 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:30,837 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:30,837 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:30,837 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:30,837 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:30,837 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:30,846 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/B/7b0f9b4518e5449dac6aa6827c7bb262, entries=150, sequenceid=54, filesize=11.7 K 2024-11-17T15:39:30,848 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/C/b5f0237d91ac4bdb9d42b90f1f9ed19a as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/C/b5f0237d91ac4bdb9d42b90f1f9ed19a 2024-11-17T15:39:30,852 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/C/b5f0237d91ac4bdb9d42b90f1f9ed19a, entries=150, sequenceid=54, filesize=11.7 K 2024-11-17T15:39:30,853 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for a682ec5138f7084a1f2f9cd3afa94497 in 176ms, sequenceid=54, compaction requested=true 2024-11-17T15:39:30,853 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a682ec5138f7084a1f2f9cd3afa94497: 2024-11-17T15:39:30,853 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a682ec5138f7084a1f2f9cd3afa94497:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T15:39:30,853 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T15:39:30,853 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a682ec5138f7084a1f2f9cd3afa94497:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T15:39:30,853 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T15:39:30,853 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a682ec5138f7084a1f2f9cd3afa94497:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T15:39:30,853 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T15:39:30,853 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T15:39:30,854 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T15:39:30,854 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110053 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T15:39:30,854 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1541): a682ec5138f7084a1f2f9cd3afa94497/A is initiating minor compaction (all files) 2024-11-17T15:39:30,855 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a682ec5138f7084a1f2f9cd3afa94497/A in TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497. 2024-11-17T15:39:30,855 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/A/7f803d4371504753a4d9d9f9fd4d4306, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/A/9a38772ae8b349758f03f17c9a6030b4, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/A/c347b5c207114d43adf46db7ab4f0d75] into tmpdir=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp, totalSize=107.5 K 2024-11-17T15:39:30,855 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497. 2024-11-17T15:39:30,855 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497. files: [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/A/7f803d4371504753a4d9d9f9fd4d4306, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/A/9a38772ae8b349758f03f17c9a6030b4, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/A/c347b5c207114d43adf46db7ab4f0d75] 2024-11-17T15:39:30,855 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T15:39:30,855 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1541): a682ec5138f7084a1f2f9cd3afa94497/B is initiating minor compaction (all files) 2024-11-17T15:39:30,855 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a682ec5138f7084a1f2f9cd3afa94497/B in TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497. 2024-11-17T15:39:30,855 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/B/e6153b84a31b4f71a64111af4a821b0f, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/B/558ab7c6bb174c94a08e7cf77978faab, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/B/7b0f9b4518e5449dac6aa6827c7bb262] into tmpdir=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp, totalSize=35.2 K 2024-11-17T15:39:30,856 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting e6153b84a31b4f71a64111af4a821b0f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731857968841 2024-11-17T15:39:30,856 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7f803d4371504753a4d9d9f9fd4d4306, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731857968841 2024-11-17T15:39:30,857 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting 558ab7c6bb174c94a08e7cf77978faab, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=42, earliestPutTs=1731857968875 2024-11-17T15:39:30,857 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9a38772ae8b349758f03f17c9a6030b4, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=42, earliestPutTs=1731857968875 2024-11-17T15:39:30,857 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting 7b0f9b4518e5449dac6aa6827c7bb262, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731857970042 2024-11-17T15:39:30,858 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting c347b5c207114d43adf46db7ab4f0d75, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731857970041 2024-11-17T15:39:30,874 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a682ec5138f7084a1f2f9cd3afa94497#B#compaction#458 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T15:39:30,875 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/B/53559797764748e496cd4b4a2681c986 is 50, key is test_row_0/B:col10/1731857970042/Put/seqid=0 2024-11-17T15:39:30,879 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a682ec5138f7084a1f2f9cd3afa94497] 2024-11-17T15:39:30,885 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411171097ab56af9c46a7b944d3c74b6a0afd_a682ec5138f7084a1f2f9cd3afa94497 store=[table=TestAcidGuarantees family=A region=a682ec5138f7084a1f2f9cd3afa94497] 2024-11-17T15:39:30,887 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411171097ab56af9c46a7b944d3c74b6a0afd_a682ec5138f7084a1f2f9cd3afa94497, store=[table=TestAcidGuarantees family=A region=a682ec5138f7084a1f2f9cd3afa94497] 2024-11-17T15:39:30,888 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411171097ab56af9c46a7b944d3c74b6a0afd_a682ec5138f7084a1f2f9cd3afa94497 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a682ec5138f7084a1f2f9cd3afa94497] 2024-11-17T15:39:30,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742366_1542 (size=12104) 2024-11-17T15:39:30,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742367_1543 (size=4469) 2024-11-17T15:39:30,912 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a682ec5138f7084a1f2f9cd3afa94497#A#compaction#459 average throughput is 0.74 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T15:39:30,913 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/A/0b280d5d14634e7c99536f38c498bc99 is 175, key is test_row_0/A:col10/1731857970042/Put/seqid=0 2024-11-17T15:39:30,940 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742368_1544 (size=31058) 2024-11-17T15:39:30,947 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/A/0b280d5d14634e7c99536f38c498bc99 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/A/0b280d5d14634e7c99536f38c498bc99 2024-11-17T15:39:30,953 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a682ec5138f7084a1f2f9cd3afa94497/A of a682ec5138f7084a1f2f9cd3afa94497 into 0b280d5d14634e7c99536f38c498bc99(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T15:39:30,953 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a682ec5138f7084a1f2f9cd3afa94497: 2024-11-17T15:39:30,953 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., storeName=a682ec5138f7084a1f2f9cd3afa94497/A, priority=13, startTime=1731857970853; duration=0sec 2024-11-17T15:39:30,954 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T15:39:30,954 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a682ec5138f7084a1f2f9cd3afa94497:A 2024-11-17T15:39:30,954 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T15:39:30,955 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T15:39:30,955 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1541): a682ec5138f7084a1f2f9cd3afa94497/C is initiating minor compaction (all files) 2024-11-17T15:39:30,956 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a682ec5138f7084a1f2f9cd3afa94497/C in TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497. 2024-11-17T15:39:30,956 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/C/e5e5127973af48a1b821b34f6fb3c536, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/C/64c61bc118674d349cf0ea1ad693fdc4, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/C/b5f0237d91ac4bdb9d42b90f1f9ed19a] into tmpdir=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp, totalSize=35.2 K 2024-11-17T15:39:30,956 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting e5e5127973af48a1b821b34f6fb3c536, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731857968841 2024-11-17T15:39:30,957 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 64c61bc118674d349cf0ea1ad693fdc4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=42, earliestPutTs=1731857968875 2024-11-17T15:39:30,957 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting b5f0237d91ac4bdb9d42b90f1f9ed19a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731857970042 2024-11-17T15:39:30,968 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a682ec5138f7084a1f2f9cd3afa94497#C#compaction#460 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T15:39:30,968 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/C/509b44c253af4543a1505011ae1c5251 is 50, key is test_row_0/C:col10/1731857970042/Put/seqid=0 2024-11-17T15:39:30,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-17T15:39:30,983 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T15:39:30,984 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T15:39:30,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] procedure2.ProcedureExecutor(1139): Stored pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees 2024-11-17T15:39:30,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-17T15:39:30,986 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T15:39:30,987 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T15:39:30,987 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=152, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T15:39:30,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742369_1545 (size=12104) 2024-11-17T15:39:30,998 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/C/509b44c253af4543a1505011ae1c5251 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/C/509b44c253af4543a1505011ae1c5251 2024-11-17T15:39:31,003 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a682ec5138f7084a1f2f9cd3afa94497/C of a682ec5138f7084a1f2f9cd3afa94497 into 509b44c253af4543a1505011ae1c5251(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T15:39:31,003 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a682ec5138f7084a1f2f9cd3afa94497: 2024-11-17T15:39:31,003 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., storeName=a682ec5138f7084a1f2f9cd3afa94497/C, priority=13, startTime=1731857970853; duration=0sec 2024-11-17T15:39:31,003 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T15:39:31,004 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a682ec5138f7084a1f2f9cd3afa94497:C 2024-11-17T15:39:31,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(8855): Flush requested on a682ec5138f7084a1f2f9cd3afa94497 2024-11-17T15:39:31,037 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T15:39:31,037 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a682ec5138f7084a1f2f9cd3afa94497 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-17T15:39:31,044 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117f0ee61d60418423d96979c68eb32a08b_a682ec5138f7084a1f2f9cd3afa94497 is 50, key is test_row_0/A:col10/1731857970719/Put/seqid=0 2024-11-17T15:39:31,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742370_1546 (size=14594) 2024-11-17T15:39:31,048 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:31,052 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117f0ee61d60418423d96979c68eb32a08b_a682ec5138f7084a1f2f9cd3afa94497 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117f0ee61d60418423d96979c68eb32a08b_a682ec5138f7084a1f2f9cd3afa94497 2024-11-17T15:39:31,054 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/A/322cb0bcf1ed433eb0b62a26680ad739, store: [table=TestAcidGuarantees family=A region=a682ec5138f7084a1f2f9cd3afa94497] 2024-11-17T15:39:31,054 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/A/322cb0bcf1ed433eb0b62a26680ad739 is 175, key is test_row_0/A:col10/1731857970719/Put/seqid=0 2024-11-17T15:39:31,055 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:31,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57310 deadline: 1731858031049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:31,055 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:31,055 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:31,055 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:31,055 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:31,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57380 deadline: 1731858031050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:31,056 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:31,056 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:31,056 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:31,060 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:31,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57288 deadline: 1731858031054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:31,060 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:31,060 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:31,060 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:31,060 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:31,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57374 deadline: 1731858031055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:31,060 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:31,061 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:31,061 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:31,062 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:31,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57332 deadline: 1731858031057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:31,062 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:31,062 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:31,062 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:31,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742371_1547 (size=39549) 2024-11-17T15:39:31,071 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=81, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/A/322cb0bcf1ed433eb0b62a26680ad739 2024-11-17T15:39:31,088 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/B/f9e1b0a87b6040bb963cd7ee8cd0aae9 is 50, key is test_row_0/B:col10/1731857970719/Put/seqid=0 2024-11-17T15:39:31,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-17T15:39:31,104 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742372_1548 (size=12001) 2024-11-17T15:39:31,115 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=81 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/B/f9e1b0a87b6040bb963cd7ee8cd0aae9 2024-11-17T15:39:31,126 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/C/d15d0151925c461ba23a2bebd6fc4ed9 is 50, key is test_row_0/C:col10/1731857970719/Put/seqid=0 2024-11-17T15:39:31,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742373_1549 (size=12001) 2024-11-17T15:39:31,140 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-17T15:39:31,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497. 2024-11-17T15:39:31,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497. as already flushing 2024-11-17T15:39:31,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497. 2024-11-17T15:39:31,141 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => a682ec5138f7084a1f2f9cd3afa94497, NAME => 'TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:39:31,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => a682ec5138f7084a1f2f9cd3afa94497, NAME => 'TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:39:31,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a682ec5138f7084a1f2f9cd3afa94497, NAME => 'TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a682ec5138f7084a1f2f9cd3afa94497, NAME => 'TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:39:31,165 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:31,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57310 deadline: 1731858031163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:31,165 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:31,165 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:31,165 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:31,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57380 deadline: 1731858031163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:31,165 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:31,166 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:31,166 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:31,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57374 deadline: 1731858031163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:31,166 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:31,166 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:31,166 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:31,166 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:31,166 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:31,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57288 deadline: 1731858031163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:31,166 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:31,166 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:31,166 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:31,166 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:31,176 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:31,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57332 deadline: 1731858031173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:31,177 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:31,177 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:31,177 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:31,292 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-17T15:39:31,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497. 2024-11-17T15:39:31,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497. as already flushing 2024-11-17T15:39:31,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497. 2024-11-17T15:39:31,293 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => a682ec5138f7084a1f2f9cd3afa94497, NAME => 'TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:39:31,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => a682ec5138f7084a1f2f9cd3afa94497, NAME => 'TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:39:31,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a682ec5138f7084a1f2f9cd3afa94497, NAME => 'TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a682ec5138f7084a1f2f9cd3afa94497, NAME => 'TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:39:31,298 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/B/53559797764748e496cd4b4a2681c986 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/B/53559797764748e496cd4b4a2681c986 2024-11-17T15:39:31,302 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a682ec5138f7084a1f2f9cd3afa94497/B of a682ec5138f7084a1f2f9cd3afa94497 into 53559797764748e496cd4b4a2681c986(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T15:39:31,302 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a682ec5138f7084a1f2f9cd3afa94497: 2024-11-17T15:39:31,302 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., storeName=a682ec5138f7084a1f2f9cd3afa94497/B, priority=13, startTime=1731857970853; duration=0sec 2024-11-17T15:39:31,302 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T15:39:31,302 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a682ec5138f7084a1f2f9cd3afa94497:B 2024-11-17T15:39:31,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-17T15:39:31,376 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:31,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57310 deadline: 1731858031373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:31,376 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:31,377 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:31,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57380 deadline: 1731858031373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:31,377 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:31,377 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:31,377 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:31,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57288 deadline: 1731858031373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:31,377 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:31,377 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:31,377 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:31,377 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:31,377 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:31,377 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:31,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57374 deadline: 1731858031373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:31,377 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:31,378 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:31,378 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:31,378 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:31,385 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:31,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57332 deadline: 1731858031383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:31,386 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:31,386 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:31,386 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:31,446 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-17T15:39:31,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497. 2024-11-17T15:39:31,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497. as already flushing 2024-11-17T15:39:31,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497. 2024-11-17T15:39:31,446 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => a682ec5138f7084a1f2f9cd3afa94497, NAME => 'TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:39:31,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => a682ec5138f7084a1f2f9cd3afa94497, NAME => 'TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:39:31,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a682ec5138f7084a1f2f9cd3afa94497, NAME => 'TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a682ec5138f7084a1f2f9cd3afa94497, NAME => 'TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:39:31,538 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=81 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/C/d15d0151925c461ba23a2bebd6fc4ed9 2024-11-17T15:39:31,542 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/A/322cb0bcf1ed433eb0b62a26680ad739 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/A/322cb0bcf1ed433eb0b62a26680ad739 2024-11-17T15:39:31,545 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/A/322cb0bcf1ed433eb0b62a26680ad739, entries=200, sequenceid=81, filesize=38.6 K 2024-11-17T15:39:31,546 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/B/f9e1b0a87b6040bb963cd7ee8cd0aae9 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/B/f9e1b0a87b6040bb963cd7ee8cd0aae9 2024-11-17T15:39:31,549 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/B/f9e1b0a87b6040bb963cd7ee8cd0aae9, entries=150, sequenceid=81, filesize=11.7 K 2024-11-17T15:39:31,551 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/C/d15d0151925c461ba23a2bebd6fc4ed9 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/C/d15d0151925c461ba23a2bebd6fc4ed9 2024-11-17T15:39:31,554 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/C/d15d0151925c461ba23a2bebd6fc4ed9, entries=150, sequenceid=81, filesize=11.7 K 2024-11-17T15:39:31,556 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=60.38 KB/61830 for a682ec5138f7084a1f2f9cd3afa94497 in 519ms, sequenceid=81, compaction requested=false 2024-11-17T15:39:31,556 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a682ec5138f7084a1f2f9cd3afa94497: 2024-11-17T15:39:31,598 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-17T15:39:31,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497. 2024-11-17T15:39:31,599 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2902): Flushing a682ec5138f7084a1f2f9cd3afa94497 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-17T15:39:31,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117d3463b31c3ec47beae12dd5e54822790_a682ec5138f7084a1f2f9cd3afa94497 is 50, key is test_row_0/A:col10/1731857971048/Put/seqid=0 2024-11-17T15:39:31,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742374_1550 (size=12154) 2024-11-17T15:39:31,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-17T15:39:31,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(8855): Flush requested on a682ec5138f7084a1f2f9cd3afa94497 2024-11-17T15:39:31,685 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497. as already flushing 2024-11-17T15:39:31,717 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:31,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57380 deadline: 1731858031710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:31,717 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:31,717 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:31,717 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:31,717 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:31,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57374 deadline: 1731858031711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:31,718 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:31,718 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:31,718 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:31,718 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:31,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57310 deadline: 1731858031712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:31,718 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:31,718 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:31,718 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:31,718 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:31,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57332 deadline: 1731858031713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:31,718 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:31,718 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:31,718 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:31,720 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:31,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57288 deadline: 1731858031716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:31,720 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:31,720 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:31,720 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:31,827 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:31,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57310 deadline: 1731858031823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:31,827 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:31,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57380 deadline: 1731858031823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:31,827 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:31,827 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:31,827 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:31,827 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:31,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57332 deadline: 1731858031823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:31,828 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:31,828 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:31,828 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:31,828 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:31,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57374 deadline: 1731858031823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:31,828 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:31,828 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:31,828 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:31,828 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:31,828 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:31,828 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:31,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57288 deadline: 1731858031823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:31,828 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:31,828 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:31,828 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:31,828 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:32,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:32,013 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117d3463b31c3ec47beae12dd5e54822790_a682ec5138f7084a1f2f9cd3afa94497 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117d3463b31c3ec47beae12dd5e54822790_a682ec5138f7084a1f2f9cd3afa94497 2024-11-17T15:39:32,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/A/30e1907f08e648819b7622b7594e1f17, store: [table=TestAcidGuarantees family=A region=a682ec5138f7084a1f2f9cd3afa94497] 2024-11-17T15:39:32,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/A/30e1907f08e648819b7622b7594e1f17 is 175, key is test_row_0/A:col10/1731857971048/Put/seqid=0 2024-11-17T15:39:32,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742375_1551 (size=30955) 2024-11-17T15:39:32,020 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=94, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/A/30e1907f08e648819b7622b7594e1f17 2024-11-17T15:39:32,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/B/16341dad9d3d415da4c870717af2a580 is 50, key is test_row_0/B:col10/1731857971048/Put/seqid=0 2024-11-17T15:39:32,036 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:32,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57380 deadline: 1731858032033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:32,037 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:32,037 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:32,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57374 deadline: 1731858032033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:32,037 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:32,037 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:32,037 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:32,037 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:32,037 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:32,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57288 deadline: 1731858032034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:32,037 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:32,037 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:32,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57310 deadline: 1731858032034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:32,037 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:32,038 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:32,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57332 deadline: 1731858032034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:32,038 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:32,038 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:32,038 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:32,038 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:32,038 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:32,038 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:32,038 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:32,038 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:32,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742376_1552 (size=12001) 2024-11-17T15:39:32,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-17T15:39:32,346 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:32,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57374 deadline: 1731858032343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:32,346 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:32,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57380 deadline: 1731858032343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:32,346 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:32,346 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:32,346 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:32,346 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:32,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57288 deadline: 1731858032343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:32,347 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:32,347 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:32,347 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:32,347 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:32,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57332 deadline: 1731858032343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:32,347 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:32,347 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:32,347 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:32,347 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:32,347 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:32,347 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:32,347 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:32,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57310 deadline: 1731858032343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:32,347 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:32,347 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:32,347 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:32,443 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/B/16341dad9d3d415da4c870717af2a580 2024-11-17T15:39:32,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/C/b10abbbc7aa1433daa4410a3d07f4769 is 50, key is test_row_0/C:col10/1731857971048/Put/seqid=0 2024-11-17T15:39:32,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742377_1553 (size=12001) 2024-11-17T15:39:32,853 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:32,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57374 deadline: 1731858032853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:32,854 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:32,854 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:32,854 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:32,856 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:32,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57380 deadline: 1731858032853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:32,856 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:32,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57332 deadline: 1731858032853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:32,857 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:32,857 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:32,857 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:32,857 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:32,857 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:32,857 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:32,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57310 deadline: 1731858032853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:32,857 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:32,857 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:32,857 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:32,857 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:32,858 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:32,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57288 deadline: 1731858032853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:32,858 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:32,858 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:32,858 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:32,860 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/C/b10abbbc7aa1433daa4410a3d07f4769 2024-11-17T15:39:32,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/A/30e1907f08e648819b7622b7594e1f17 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/A/30e1907f08e648819b7622b7594e1f17 2024-11-17T15:39:32,867 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/A/30e1907f08e648819b7622b7594e1f17, entries=150, sequenceid=94, filesize=30.2 K 2024-11-17T15:39:32,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/B/16341dad9d3d415da4c870717af2a580 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/B/16341dad9d3d415da4c870717af2a580 2024-11-17T15:39:32,872 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/B/16341dad9d3d415da4c870717af2a580, entries=150, sequenceid=94, filesize=11.7 K 2024-11-17T15:39:32,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/C/b10abbbc7aa1433daa4410a3d07f4769 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/C/b10abbbc7aa1433daa4410a3d07f4769 2024-11-17T15:39:32,875 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/C/b10abbbc7aa1433daa4410a3d07f4769, entries=150, sequenceid=94, filesize=11.7 K 2024-11-17T15:39:32,876 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for a682ec5138f7084a1f2f9cd3afa94497 in 1277ms, sequenceid=94, compaction requested=true 2024-11-17T15:39:32,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2603): Flush status journal for a682ec5138f7084a1f2f9cd3afa94497: 2024-11-17T15:39:32,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497. 2024-11-17T15:39:32,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=153 2024-11-17T15:39:32,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.HMaster(4169): Remote procedure done, pid=153 2024-11-17T15:39:32,880 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=152 2024-11-17T15:39:32,880 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8910 sec 2024-11-17T15:39:32,882 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees in 1.8960 sec 2024-11-17T15:39:33,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-17T15:39:33,134 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T15:39:33,134 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T15:39:33,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] procedure2.ProcedureExecutor(1139): Stored pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees 2024-11-17T15:39:33,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-17T15:39:33,136 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T15:39:33,137 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T15:39:33,137 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T15:39:33,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-17T15:39:33,289 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-17T15:39:33,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497. 2024-11-17T15:39:33,289 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2902): Flushing a682ec5138f7084a1f2f9cd3afa94497 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-17T15:39:33,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117cfb4d07d1c6b43a7b51c520d6f1aa66d_a682ec5138f7084a1f2f9cd3afa94497 is 50, key is test_row_0/A:col10/1731857971709/Put/seqid=0 2024-11-17T15:39:33,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742378_1554 (size=12154) 2024-11-17T15:39:33,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-17T15:39:33,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:33,705 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117cfb4d07d1c6b43a7b51c520d6f1aa66d_a682ec5138f7084a1f2f9cd3afa94497 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117cfb4d07d1c6b43a7b51c520d6f1aa66d_a682ec5138f7084a1f2f9cd3afa94497 2024-11-17T15:39:33,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/A/2460528174d34b39bc887223334634b2, store: [table=TestAcidGuarantees family=A region=a682ec5138f7084a1f2f9cd3afa94497] 2024-11-17T15:39:33,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/A/2460528174d34b39bc887223334634b2 is 175, key is test_row_0/A:col10/1731857971709/Put/seqid=0 2024-11-17T15:39:33,711 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742379_1555 (size=30955) 2024-11-17T15:39:33,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-17T15:39:33,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(8855): Flush requested on a682ec5138f7084a1f2f9cd3afa94497 2024-11-17T15:39:33,867 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497. as already flushing 2024-11-17T15:39:33,879 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:33,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57380 deadline: 1731858033875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:33,879 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:33,879 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:33,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57288 deadline: 1731858033875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:33,880 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:33,880 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:33,880 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:33,880 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:33,880 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:33,882 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:33,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57374 deadline: 1731858033877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:33,883 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:33,883 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:33,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57332 deadline: 1731858033878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:33,883 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:33,883 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:33,883 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:33,883 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:33,883 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:33,884 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:33,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57310 deadline: 1731858033879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:33,884 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:33,884 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:33,885 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:33,985 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:33,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57380 deadline: 1731858033983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:33,985 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:33,985 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:33,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57288 deadline: 1731858033983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:33,985 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:33,985 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:33,986 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:33,986 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:33,986 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:33,996 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:33,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57374 deadline: 1731858033993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:33,997 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:33,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57332 deadline: 1731858033993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:33,997 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:33,997 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:33,997 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:33,997 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:33,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57310 deadline: 1731858033993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:33,997 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:33,997 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:33,997 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:33,997 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:33,997 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:33,997 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:34,112 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=118, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/A/2460528174d34b39bc887223334634b2 2024-11-17T15:39:34,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/B/9bdb77f2b6614f818af43f71c5ea59ef is 50, key is test_row_0/B:col10/1731857971709/Put/seqid=0 2024-11-17T15:39:34,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742380_1556 (size=12001) 2024-11-17T15:39:34,196 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:34,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57380 deadline: 1731858034193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:34,196 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:34,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57288 deadline: 1731858034193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:34,197 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:34,197 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:34,197 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:34,197 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:34,197 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:34,197 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:34,206 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:34,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57374 deadline: 1731858034203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:34,207 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:34,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57332 deadline: 1731858034203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:34,207 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:34,207 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:34,207 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:34,207 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:34,207 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:34,207 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:34,207 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:34,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57310 deadline: 1731858034204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:34,207 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:34,207 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:34,207 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:34,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-17T15:39:34,506 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:34,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57288 deadline: 1731858034503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:34,507 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:34,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57380 deadline: 1731858034503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:34,507 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:34,507 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:34,507 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:34,507 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:34,507 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:34,507 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:34,517 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:34,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57374 deadline: 1731858034513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:34,517 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:34,517 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:34,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57332 deadline: 1731858034513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:34,517 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:34,517 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:34,518 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:34,518 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:34,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57310 deadline: 1731858034513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:34,518 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:34,518 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:34,518 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:34,518 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:34,518 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:34,526 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/B/9bdb77f2b6614f818af43f71c5ea59ef 2024-11-17T15:39:34,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/C/72ba348337814dc690a03bfc5b238c08 is 50, key is test_row_0/C:col10/1731857971709/Put/seqid=0 2024-11-17T15:39:34,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742381_1557 (size=12001) 2024-11-17T15:39:34,536 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/C/72ba348337814dc690a03bfc5b238c08 2024-11-17T15:39:34,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/A/2460528174d34b39bc887223334634b2 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/A/2460528174d34b39bc887223334634b2 2024-11-17T15:39:34,555 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/A/2460528174d34b39bc887223334634b2, entries=150, sequenceid=118, filesize=30.2 K 2024-11-17T15:39:34,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/B/9bdb77f2b6614f818af43f71c5ea59ef as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/B/9bdb77f2b6614f818af43f71c5ea59ef 2024-11-17T15:39:34,559 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/B/9bdb77f2b6614f818af43f71c5ea59ef, entries=150, sequenceid=118, filesize=11.7 K 2024-11-17T15:39:34,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/C/72ba348337814dc690a03bfc5b238c08 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/C/72ba348337814dc690a03bfc5b238c08 2024-11-17T15:39:34,564 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/C/72ba348337814dc690a03bfc5b238c08, entries=150, sequenceid=118, filesize=11.7 K 2024-11-17T15:39:34,565 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for a682ec5138f7084a1f2f9cd3afa94497 in 1276ms, sequenceid=118, compaction requested=true 2024-11-17T15:39:34,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2603): Flush status journal for a682ec5138f7084a1f2f9cd3afa94497: 2024-11-17T15:39:34,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497. 2024-11-17T15:39:34,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=155 2024-11-17T15:39:34,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.HMaster(4169): Remote procedure done, pid=155 2024-11-17T15:39:34,568 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=154 2024-11-17T15:39:34,568 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4290 sec 2024-11-17T15:39:34,569 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees in 1.4340 sec 2024-11-17T15:39:35,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(8855): Flush requested on a682ec5138f7084a1f2f9cd3afa94497 2024-11-17T15:39:35,017 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T15:39:35,017 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a682ec5138f7084a1f2f9cd3afa94497 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-17T15:39:35,026 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111799daa4da7d0645419010868f434e2fc7_a682ec5138f7084a1f2f9cd3afa94497 is 50, key is test_row_0/A:col10/1731857973874/Put/seqid=0 2024-11-17T15:39:35,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742382_1558 (size=14694) 2024-11-17T15:39:35,037 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:35,044 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111799daa4da7d0645419010868f434e2fc7_a682ec5138f7084a1f2f9cd3afa94497 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111799daa4da7d0645419010868f434e2fc7_a682ec5138f7084a1f2f9cd3afa94497 2024-11-17T15:39:35,044 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/A/bc990395726143668c889819bd9e605c, store: [table=TestAcidGuarantees family=A region=a682ec5138f7084a1f2f9cd3afa94497] 2024-11-17T15:39:35,045 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/A/bc990395726143668c889819bd9e605c is 175, key is test_row_0/A:col10/1731857973874/Put/seqid=0 2024-11-17T15:39:35,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742383_1559 (size=39649) 2024-11-17T15:39:35,050 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=132, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/A/bc990395726143668c889819bd9e605c 2024-11-17T15:39:35,056 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/B/2f91faea19ac40edb92244a7bfec2b46 is 50, key is test_row_0/B:col10/1731857973874/Put/seqid=0 2024-11-17T15:39:35,059 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:35,059 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:35,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742384_1560 (size=12051) 2024-11-17T15:39:35,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57288 deadline: 1731858035048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:35,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57310 deadline: 1731858035049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:35,059 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:35,059 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:35,059 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:35,059 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:35,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57374 deadline: 1731858035050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:35,060 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/B/2f91faea19ac40edb92244a7bfec2b46 2024-11-17T15:39:35,060 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:35,060 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:35,060 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:35,060 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:35,060 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:35,060 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:35,065 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:35,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57380 deadline: 1731858035059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:35,065 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:35,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57332 deadline: 1731858035059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:35,065 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:35,065 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:35,065 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:35,066 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:35,066 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:35,066 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:35,066 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/C/690db289c6ff4559bf0f9ce78f77c41c is 50, key is test_row_0/C:col10/1731857973874/Put/seqid=0 2024-11-17T15:39:35,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742385_1561 (size=12051) 2024-11-17T15:39:35,069 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/C/690db289c6ff4559bf0f9ce78f77c41c 2024-11-17T15:39:35,072 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/A/bc990395726143668c889819bd9e605c as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/A/bc990395726143668c889819bd9e605c 2024-11-17T15:39:35,075 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/A/bc990395726143668c889819bd9e605c, entries=200, sequenceid=132, filesize=38.7 K 2024-11-17T15:39:35,076 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/B/2f91faea19ac40edb92244a7bfec2b46 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/B/2f91faea19ac40edb92244a7bfec2b46 2024-11-17T15:39:35,079 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/B/2f91faea19ac40edb92244a7bfec2b46, entries=150, sequenceid=132, filesize=11.8 K 2024-11-17T15:39:35,079 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/C/690db289c6ff4559bf0f9ce78f77c41c as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/C/690db289c6ff4559bf0f9ce78f77c41c 2024-11-17T15:39:35,083 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/C/690db289c6ff4559bf0f9ce78f77c41c, entries=150, sequenceid=132, filesize=11.8 K 2024-11-17T15:39:35,083 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for a682ec5138f7084a1f2f9cd3afa94497 in 66ms, sequenceid=132, compaction requested=true 2024-11-17T15:39:35,083 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a682ec5138f7084a1f2f9cd3afa94497: 2024-11-17T15:39:35,083 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a682ec5138f7084a1f2f9cd3afa94497:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T15:39:35,084 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T15:39:35,084 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-17T15:39:35,084 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-17T15:39:35,084 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a682ec5138f7084a1f2f9cd3afa94497:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T15:39:35,084 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T15:39:35,084 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a682ec5138f7084a1f2f9cd3afa94497:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T15:39:35,084 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T15:39:35,092 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 172166 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-17T15:39:35,092 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 60158 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-17T15:39:35,092 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1541): a682ec5138f7084a1f2f9cd3afa94497/B is initiating minor compaction (all files) 2024-11-17T15:39:35,092 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1541): a682ec5138f7084a1f2f9cd3afa94497/A is initiating minor compaction (all files) 2024-11-17T15:39:35,092 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a682ec5138f7084a1f2f9cd3afa94497/A in TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497. 2024-11-17T15:39:35,092 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a682ec5138f7084a1f2f9cd3afa94497/B in TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497. 2024-11-17T15:39:35,092 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/A/0b280d5d14634e7c99536f38c498bc99, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/A/322cb0bcf1ed433eb0b62a26680ad739, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/A/30e1907f08e648819b7622b7594e1f17, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/A/2460528174d34b39bc887223334634b2, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/A/bc990395726143668c889819bd9e605c] into tmpdir=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp, totalSize=168.1 K 2024-11-17T15:39:35,092 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/B/53559797764748e496cd4b4a2681c986, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/B/f9e1b0a87b6040bb963cd7ee8cd0aae9, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/B/16341dad9d3d415da4c870717af2a580, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/B/9bdb77f2b6614f818af43f71c5ea59ef, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/B/2f91faea19ac40edb92244a7bfec2b46] into tmpdir=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp, totalSize=58.7 K 2024-11-17T15:39:35,092 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=11 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497. 2024-11-17T15:39:35,093 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497. files: [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/A/0b280d5d14634e7c99536f38c498bc99, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/A/322cb0bcf1ed433eb0b62a26680ad739, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/A/30e1907f08e648819b7622b7594e1f17, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/A/2460528174d34b39bc887223334634b2, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/A/bc990395726143668c889819bd9e605c] 2024-11-17T15:39:35,093 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0b280d5d14634e7c99536f38c498bc99, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731857970042 2024-11-17T15:39:35,093 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting 53559797764748e496cd4b4a2681c986, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731857970042 2024-11-17T15:39:35,093 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 322cb0bcf1ed433eb0b62a26680ad739, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1731857970707 2024-11-17T15:39:35,093 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting f9e1b0a87b6040bb963cd7ee8cd0aae9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1731857970716 2024-11-17T15:39:35,096 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 30e1907f08e648819b7622b7594e1f17, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731857971048 2024-11-17T15:39:35,096 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting 16341dad9d3d415da4c870717af2a580, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731857971048 2024-11-17T15:39:35,096 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2460528174d34b39bc887223334634b2, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1731857971709 2024-11-17T15:39:35,096 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting bc990395726143668c889819bd9e605c, keycount=200, bloomtype=ROW, size=38.7 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731857973874 2024-11-17T15:39:35,097 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting 9bdb77f2b6614f818af43f71c5ea59ef, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1731857971709 2024-11-17T15:39:35,097 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting 2f91faea19ac40edb92244a7bfec2b46, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731857973874 2024-11-17T15:39:35,108 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a682ec5138f7084a1f2f9cd3afa94497] 2024-11-17T15:39:35,110 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111722b1cc9fb8c64a8cb30eccf40a6af615_a682ec5138f7084a1f2f9cd3afa94497 store=[table=TestAcidGuarantees family=A region=a682ec5138f7084a1f2f9cd3afa94497] 2024-11-17T15:39:35,113 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a682ec5138f7084a1f2f9cd3afa94497#B#compaction#474 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T15:39:35,113 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111722b1cc9fb8c64a8cb30eccf40a6af615_a682ec5138f7084a1f2f9cd3afa94497, store=[table=TestAcidGuarantees family=A region=a682ec5138f7084a1f2f9cd3afa94497] 2024-11-17T15:39:35,113 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111722b1cc9fb8c64a8cb30eccf40a6af615_a682ec5138f7084a1f2f9cd3afa94497 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a682ec5138f7084a1f2f9cd3afa94497] 2024-11-17T15:39:35,113 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/B/5f3f51f9e48a49d0bfadc9663cec248a is 50, key is test_row_0/B:col10/1731857973874/Put/seqid=0 2024-11-17T15:39:35,135 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742387_1563 (size=12325) 2024-11-17T15:39:35,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742386_1562 (size=4469) 2024-11-17T15:39:35,151 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a682ec5138f7084a1f2f9cd3afa94497#A#compaction#473 average throughput is 0.57 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T15:39:35,151 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/A/fe806eb1dbd44354ba9f546e943ba3f7 is 175, key is test_row_0/A:col10/1731857973874/Put/seqid=0 2024-11-17T15:39:35,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(8855): Flush requested on a682ec5138f7084a1f2f9cd3afa94497 2024-11-17T15:39:35,167 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T15:39:35,167 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a682ec5138f7084a1f2f9cd3afa94497 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-17T15:39:35,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742388_1564 (size=31279) 2024-11-17T15:39:35,180 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/A/fe806eb1dbd44354ba9f546e943ba3f7 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/A/fe806eb1dbd44354ba9f546e943ba3f7 2024-11-17T15:39:35,185 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in a682ec5138f7084a1f2f9cd3afa94497/A of a682ec5138f7084a1f2f9cd3afa94497 into fe806eb1dbd44354ba9f546e943ba3f7(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T15:39:35,185 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a682ec5138f7084a1f2f9cd3afa94497: 2024-11-17T15:39:35,185 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., storeName=a682ec5138f7084a1f2f9cd3afa94497/A, priority=11, startTime=1731857975083; duration=0sec 2024-11-17T15:39:35,185 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T15:39:35,185 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a682ec5138f7084a1f2f9cd3afa94497:A 2024-11-17T15:39:35,185 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-17T15:39:35,188 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:35,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57380 deadline: 1731858035181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:35,189 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:35,189 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 60158 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-17T15:39:35,189 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:35,189 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:35,189 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1541): a682ec5138f7084a1f2f9cd3afa94497/C is initiating minor compaction (all files) 2024-11-17T15:39:35,189 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a682ec5138f7084a1f2f9cd3afa94497/C in TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497. 2024-11-17T15:39:35,189 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/C/509b44c253af4543a1505011ae1c5251, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/C/d15d0151925c461ba23a2bebd6fc4ed9, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/C/b10abbbc7aa1433daa4410a3d07f4769, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/C/72ba348337814dc690a03bfc5b238c08, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/C/690db289c6ff4559bf0f9ce78f77c41c] into tmpdir=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp, totalSize=58.7 K 2024-11-17T15:39:35,190 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 509b44c253af4543a1505011ae1c5251, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731857970042 2024-11-17T15:39:35,190 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting d15d0151925c461ba23a2bebd6fc4ed9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1731857970716 2024-11-17T15:39:35,190 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting b10abbbc7aa1433daa4410a3d07f4769, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731857971048 2024-11-17T15:39:35,191 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 72ba348337814dc690a03bfc5b238c08, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1731857971709 2024-11-17T15:39:35,191 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 690db289c6ff4559bf0f9ce78f77c41c, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731857973874 2024-11-17T15:39:35,196 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:35,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57332 deadline: 1731858035183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:35,196 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:35,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57288 deadline: 1731858035185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:35,196 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:35,196 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:35,196 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:35,197 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:35,197 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:35,197 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:35,197 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:35,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57310 deadline: 1731858035189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:35,198 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:35,198 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:35,198 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:35,198 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:35,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57374 deadline: 1731858035189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:35,198 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:35,198 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:35,198 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:35,203 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411172580b345c34c442c95b6a4afe066eb1d_a682ec5138f7084a1f2f9cd3afa94497 is 50, key is test_row_0/A:col10/1731857975166/Put/seqid=0 2024-11-17T15:39:35,213 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a682ec5138f7084a1f2f9cd3afa94497#C#compaction#476 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T15:39:35,214 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/C/cad4210b0b6d4656b6ec08af997136b5 is 50, key is test_row_0/C:col10/1731857973874/Put/seqid=0 2024-11-17T15:39:35,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742389_1565 (size=14794) 2024-11-17T15:39:35,253 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:35,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742390_1566 (size=12325) 2024-11-17T15:39:35,258 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411172580b345c34c442c95b6a4afe066eb1d_a682ec5138f7084a1f2f9cd3afa94497 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411172580b345c34c442c95b6a4afe066eb1d_a682ec5138f7084a1f2f9cd3afa94497 2024-11-17T15:39:35,259 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/A/3604334c372a467ab844d06f04087498, store: [table=TestAcidGuarantees family=A region=a682ec5138f7084a1f2f9cd3afa94497] 2024-11-17T15:39:35,260 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/A/3604334c372a467ab844d06f04087498 is 175, key is test_row_0/A:col10/1731857975166/Put/seqid=0 2024-11-17T15:39:35,266 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742391_1567 (size=39749) 2024-11-17T15:39:35,267 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=156, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/A/3604334c372a467ab844d06f04087498 2024-11-17T15:39:35,270 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/C/cad4210b0b6d4656b6ec08af997136b5 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/C/cad4210b0b6d4656b6ec08af997136b5 2024-11-17T15:39:35,278 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/B/376b2eec641f4db18b5483d68ec80df1 is 50, key is test_row_0/B:col10/1731857975166/Put/seqid=0 2024-11-17T15:39:35,281 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in a682ec5138f7084a1f2f9cd3afa94497/C of a682ec5138f7084a1f2f9cd3afa94497 into cad4210b0b6d4656b6ec08af997136b5(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T15:39:35,281 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a682ec5138f7084a1f2f9cd3afa94497: 2024-11-17T15:39:35,281 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., storeName=a682ec5138f7084a1f2f9cd3afa94497/C, priority=11, startTime=1731857975084; duration=0sec 2024-11-17T15:39:35,281 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T15:39:35,281 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a682ec5138f7084a1f2f9cd3afa94497:C 2024-11-17T15:39:35,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-17T15:39:35,283 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T15:39:35,284 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T15:39:35,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] procedure2.ProcedureExecutor(1139): Stored pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees 2024-11-17T15:39:35,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-17T15:39:35,287 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T15:39:35,288 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T15:39:35,288 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T15:39:35,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742392_1568 (size=12151) 2024-11-17T15:39:35,292 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/B/376b2eec641f4db18b5483d68ec80df1 2024-11-17T15:39:35,296 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:35,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57380 deadline: 1731858035293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:35,297 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:35,297 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:35,297 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:35,300 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/C/21ab8f1c786c40b6997dc6fa2a1cf6a4 is 50, key is test_row_0/C:col10/1731857975166/Put/seqid=0 2024-11-17T15:39:35,307 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:35,307 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:35,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57310 deadline: 1731858035303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:35,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57332 deadline: 1731858035303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:35,308 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:35,308 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:35,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57374 deadline: 1731858035303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:35,308 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:35,308 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:35,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57288 deadline: 1731858035303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:35,308 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:35,308 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:35,308 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:35,308 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:35,308 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:35,308 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:35,308 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:35,308 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:35,308 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:35,308 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:35,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742393_1569 (size=12151) 2024-11-17T15:39:35,312 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/C/21ab8f1c786c40b6997dc6fa2a1cf6a4 2024-11-17T15:39:35,319 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/A/3604334c372a467ab844d06f04087498 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/A/3604334c372a467ab844d06f04087498 2024-11-17T15:39:35,322 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/A/3604334c372a467ab844d06f04087498, entries=200, sequenceid=156, filesize=38.8 K 2024-11-17T15:39:35,329 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/B/376b2eec641f4db18b5483d68ec80df1 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/B/376b2eec641f4db18b5483d68ec80df1 2024-11-17T15:39:35,337 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/B/376b2eec641f4db18b5483d68ec80df1, entries=150, sequenceid=156, filesize=11.9 K 2024-11-17T15:39:35,338 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/C/21ab8f1c786c40b6997dc6fa2a1cf6a4 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/C/21ab8f1c786c40b6997dc6fa2a1cf6a4 2024-11-17T15:39:35,342 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/C/21ab8f1c786c40b6997dc6fa2a1cf6a4, entries=150, sequenceid=156, filesize=11.9 K 2024-11-17T15:39:35,343 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=67.09 KB/68700 for a682ec5138f7084a1f2f9cd3afa94497 in 175ms, sequenceid=156, compaction requested=false 2024-11-17T15:39:35,343 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a682ec5138f7084a1f2f9cd3afa94497: 2024-11-17T15:39:35,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-17T15:39:35,441 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-17T15:39:35,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497. 2024-11-17T15:39:35,441 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2902): Flushing a682ec5138f7084a1f2f9cd3afa94497 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-17T15:39:35,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117345e53bf303140c180594919a6fc5bd0_a682ec5138f7084a1f2f9cd3afa94497 is 50, key is test_row_0/A:col10/1731857975186/Put/seqid=0 2024-11-17T15:39:35,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742394_1570 (size=12304) 2024-11-17T15:39:35,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(8855): Flush requested on a682ec5138f7084a1f2f9cd3afa94497 2024-11-17T15:39:35,506 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497. as already flushing 2024-11-17T15:39:35,540 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/B/5f3f51f9e48a49d0bfadc9663cec248a as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/B/5f3f51f9e48a49d0bfadc9663cec248a 2024-11-17T15:39:35,541 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:35,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57310 deadline: 1731858035536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:35,542 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:35,542 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:35,542 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:35,542 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:35,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57374 deadline: 1731858035537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:35,542 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:35,542 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:35,542 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:35,544 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in a682ec5138f7084a1f2f9cd3afa94497/B of a682ec5138f7084a1f2f9cd3afa94497 into 5f3f51f9e48a49d0bfadc9663cec248a(size=12.0 K), total size for store is 23.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T15:39:35,544 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a682ec5138f7084a1f2f9cd3afa94497: 2024-11-17T15:39:35,544 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., storeName=a682ec5138f7084a1f2f9cd3afa94497/B, priority=11, startTime=1731857975084; duration=0sec 2024-11-17T15:39:35,545 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T15:39:35,545 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a682ec5138f7084a1f2f9cd3afa94497:B 2024-11-17T15:39:35,545 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:35,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57380 deadline: 1731858035538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:35,545 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:35,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57332 deadline: 1731858035540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:35,545 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:35,545 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:35,545 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:35,545 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:35,546 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:35,546 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:35,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57288 deadline: 1731858035541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:35,546 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:35,548 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:35,548 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:35,548 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:35,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-17T15:39:35,655 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:35,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57310 deadline: 1731858035653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:35,656 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:35,656 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:35,656 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:35,656 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:35,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57380 deadline: 1731858035653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:35,656 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:35,656 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:35,657 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:35,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57332 deadline: 1731858035653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:35,657 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:35,657 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:35,657 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:35,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57374 deadline: 1731858035654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:35,657 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:35,657 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:35,657 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:35,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57288 deadline: 1731858035654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:35,657 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:35,657 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:35,657 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:35,657 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:35,657 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:35,657 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:35,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:35,858 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117345e53bf303140c180594919a6fc5bd0_a682ec5138f7084a1f2f9cd3afa94497 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117345e53bf303140c180594919a6fc5bd0_a682ec5138f7084a1f2f9cd3afa94497 2024-11-17T15:39:35,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/A/415ab553c1a54754be54ca8433b84166, store: [table=TestAcidGuarantees family=A region=a682ec5138f7084a1f2f9cd3afa94497] 2024-11-17T15:39:35,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/A/415ab553c1a54754be54ca8433b84166 is 175, key is test_row_0/A:col10/1731857975186/Put/seqid=0 2024-11-17T15:39:35,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742395_1571 (size=31105) 2024-11-17T15:39:35,866 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:35,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57310 deadline: 1731858035863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:35,866 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:35,867 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:35,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57380 deadline: 1731858035863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:35,867 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:35,867 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:35,867 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:35,867 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:35,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57332 deadline: 1731858035863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:35,867 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:35,867 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:35,867 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:35,867 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:35,867 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:35,867 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:35,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57374 deadline: 1731858035864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:35,868 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:35,868 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:35,868 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:35,868 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:35,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57288 deadline: 1731858035864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:35,868 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:35,868 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:35,868 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:35,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-17T15:39:36,176 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:36,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57380 deadline: 1731858036173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:36,177 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:36,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57374 deadline: 1731858036173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:36,177 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:36,177 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:36,177 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:36,177 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:36,177 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:36,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57288 deadline: 1731858036173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:36,177 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:36,177 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:36,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57310 deadline: 1731858036173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:36,177 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:36,177 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:36,177 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:36,177 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:36,177 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:36,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57332 deadline: 1731858036174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:36,177 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:36,177 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:36,177 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:36,178 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:36,178 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:36,178 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:36,264 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=171, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/A/415ab553c1a54754be54ca8433b84166 2024-11-17T15:39:36,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/B/edcccf8a10c041d28b11d339d20d70c4 is 50, key is test_row_0/B:col10/1731857975186/Put/seqid=0 2024-11-17T15:39:36,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742396_1572 (size=12151) 2024-11-17T15:39:36,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-17T15:39:36,684 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:36,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57380 deadline: 1731858036683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:36,685 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:36,685 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:36,685 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:36,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57374 deadline: 1731858036683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:36,685 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:36,685 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:36,685 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:36,685 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:36,685 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:36,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57310 deadline: 1731858036684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:36,685 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:36,685 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:36,685 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:36,685 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:36,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57332 deadline: 1731858036684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:36,686 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:36,686 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:36,686 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:36,686 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:36,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57288 deadline: 1731858036684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:36,686 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:36,686 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:36,686 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:36,703 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/B/edcccf8a10c041d28b11d339d20d70c4 2024-11-17T15:39:36,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/C/cd225f855b884a6cbc022d4a0a571f55 is 50, key is test_row_0/C:col10/1731857975186/Put/seqid=0 2024-11-17T15:39:36,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742397_1573 (size=12151) 2024-11-17T15:39:37,118 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/C/cd225f855b884a6cbc022d4a0a571f55 2024-11-17T15:39:37,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/A/415ab553c1a54754be54ca8433b84166 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/A/415ab553c1a54754be54ca8433b84166 2024-11-17T15:39:37,129 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/A/415ab553c1a54754be54ca8433b84166, entries=150, sequenceid=171, filesize=30.4 K 2024-11-17T15:39:37,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/B/edcccf8a10c041d28b11d339d20d70c4 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/B/edcccf8a10c041d28b11d339d20d70c4 2024-11-17T15:39:37,134 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/B/edcccf8a10c041d28b11d339d20d70c4, entries=150, sequenceid=171, filesize=11.9 K 2024-11-17T15:39:37,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/C/cd225f855b884a6cbc022d4a0a571f55 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/C/cd225f855b884a6cbc022d4a0a571f55 2024-11-17T15:39:37,138 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/C/cd225f855b884a6cbc022d4a0a571f55, entries=150, sequenceid=171, filesize=11.9 K 2024-11-17T15:39:37,139 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for a682ec5138f7084a1f2f9cd3afa94497 in 1697ms, sequenceid=171, compaction requested=true 2024-11-17T15:39:37,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2603): Flush status journal for a682ec5138f7084a1f2f9cd3afa94497: 2024-11-17T15:39:37,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497. 2024-11-17T15:39:37,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=157 2024-11-17T15:39:37,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.HMaster(4169): Remote procedure done, pid=157 2024-11-17T15:39:37,143 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-11-17T15:39:37,143 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8520 sec 2024-11-17T15:39:37,146 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees in 1.8600 sec 2024-11-17T15:39:37,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-17T15:39:37,433 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T15:39:37,434 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T15:39:37,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] procedure2.ProcedureExecutor(1139): Stored pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees 2024-11-17T15:39:37,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-17T15:39:37,436 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T15:39:37,437 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T15:39:37,437 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T15:39:37,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-17T15:39:37,589 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-17T15:39:37,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497. 2024-11-17T15:39:37,589 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2902): Flushing a682ec5138f7084a1f2f9cd3afa94497 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-17T15:39:37,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117098e390e0d894b2b8a53f67c36bee6f2_a682ec5138f7084a1f2f9cd3afa94497 is 50, key is test_row_0/A:col10/1731857975532/Put/seqid=0 2024-11-17T15:39:37,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742398_1574 (size=12304) 2024-11-17T15:39:37,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(8855): Flush requested on a682ec5138f7084a1f2f9cd3afa94497 2024-11-17T15:39:37,696 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497. as already flushing 2024-11-17T15:39:37,710 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:37,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57374 deadline: 1731858037707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:37,711 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:37,711 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:37,711 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:37,715 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:37,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57288 deadline: 1731858037709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:37,715 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:37,715 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:37,715 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:37,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57332 deadline: 1731858037709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:37,715 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:37,715 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:37,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57310 deadline: 1731858037710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:37,715 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:37,715 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:37,715 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:37,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57380 deadline: 1731858037711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:37,715 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:37,716 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:37,716 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:37,716 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:37,716 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:37,716 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:37,716 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:37,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-17T15:39:37,816 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:37,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57374 deadline: 1731858037813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:37,816 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:37,816 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:37,816 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:37,826 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:37,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57288 deadline: 1731858037823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:37,826 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:37,826 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:37,826 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:37,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57332 deadline: 1731858037823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:37,826 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:37,826 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:37,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57310 deadline: 1731858037823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:37,827 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:37,827 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:37,827 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:37,827 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:37,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57380 deadline: 1731858037823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:37,827 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:37,827 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:37,827 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:37,827 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:37,827 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:37,827 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:38,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:38,008 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117098e390e0d894b2b8a53f67c36bee6f2_a682ec5138f7084a1f2f9cd3afa94497 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117098e390e0d894b2b8a53f67c36bee6f2_a682ec5138f7084a1f2f9cd3afa94497 2024-11-17T15:39:38,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/A/dc16ad2d35744a4095376e009088dabc, store: [table=TestAcidGuarantees family=A region=a682ec5138f7084a1f2f9cd3afa94497] 2024-11-17T15:39:38,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/A/dc16ad2d35744a4095376e009088dabc is 175, key is test_row_0/A:col10/1731857975532/Put/seqid=0 2024-11-17T15:39:38,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742399_1575 (size=31105) 2024-11-17T15:39:38,013 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=195, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/A/dc16ad2d35744a4095376e009088dabc 2024-11-17T15:39:38,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/B/9e065d0e15f44fec9daffa63c10ccc7d is 50, key is test_row_0/B:col10/1731857975532/Put/seqid=0 2024-11-17T15:39:38,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742400_1576 (size=12151) 2024-11-17T15:39:38,026 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/B/9e065d0e15f44fec9daffa63c10ccc7d 2024-11-17T15:39:38,026 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:38,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57374 deadline: 1731858038023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:38,027 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:38,027 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:38,027 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:38,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/C/2f12e08c211a4581a7d222a6bc31524d is 50, key is test_row_0/C:col10/1731857975532/Put/seqid=0 2024-11-17T15:39:38,036 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:38,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57288 deadline: 1731858038033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:38,036 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:38,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57310 deadline: 1731858038033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:38,037 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:38,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742401_1577 (size=12151) 2024-11-17T15:39:38,037 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:38,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57332 deadline: 1731858038033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:38,037 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:38,037 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:38,037 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:38,037 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:38,037 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:38,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57380 deadline: 1731858038034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:38,037 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:38,037 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:38,037 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:38,037 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:38,037 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:38,037 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:38,037 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:38,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-17T15:39:38,336 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:38,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57374 deadline: 1731858038333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:38,337 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:38,337 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:38,337 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:38,346 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:38,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57310 deadline: 1731858038343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:38,346 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:38,346 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:38,346 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:38,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57288 deadline: 1731858038344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:38,346 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:38,347 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:38,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57332 deadline: 1731858038344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:38,347 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:38,347 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:38,347 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:38,347 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:38,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57380 deadline: 1731858038344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:38,347 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:38,347 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:38,347 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:38,356 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:38,356 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:38,356 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:38,437 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/C/2f12e08c211a4581a7d222a6bc31524d 2024-11-17T15:39:38,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/A/dc16ad2d35744a4095376e009088dabc as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/A/dc16ad2d35744a4095376e009088dabc 2024-11-17T15:39:38,444 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/A/dc16ad2d35744a4095376e009088dabc, entries=150, sequenceid=195, filesize=30.4 K 2024-11-17T15:39:38,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/B/9e065d0e15f44fec9daffa63c10ccc7d as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/B/9e065d0e15f44fec9daffa63c10ccc7d 2024-11-17T15:39:38,448 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/B/9e065d0e15f44fec9daffa63c10ccc7d, entries=150, sequenceid=195, filesize=11.9 K 2024-11-17T15:39:38,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/C/2f12e08c211a4581a7d222a6bc31524d as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/C/2f12e08c211a4581a7d222a6bc31524d 2024-11-17T15:39:38,452 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/C/2f12e08c211a4581a7d222a6bc31524d, entries=150, sequenceid=195, filesize=11.9 K 2024-11-17T15:39:38,453 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for a682ec5138f7084a1f2f9cd3afa94497 in 864ms, sequenceid=195, compaction requested=true 2024-11-17T15:39:38,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2603): Flush status journal for a682ec5138f7084a1f2f9cd3afa94497: 2024-11-17T15:39:38,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497. 2024-11-17T15:39:38,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=159 2024-11-17T15:39:38,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.HMaster(4169): Remote procedure done, pid=159 2024-11-17T15:39:38,456 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=158 2024-11-17T15:39:38,456 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0170 sec 2024-11-17T15:39:38,458 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees in 1.0220 sec 2024-11-17T15:39:38,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-17T15:39:38,573 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T15:39:38,573 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T15:39:38,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] procedure2.ProcedureExecutor(1139): Stored pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees 2024-11-17T15:39:38,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-17T15:39:38,575 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T15:39:38,576 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T15:39:38,576 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T15:39:38,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-17T15:39:38,728 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-17T15:39:38,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497. 2024-11-17T15:39:38,728 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2902): Flushing a682ec5138f7084a1f2f9cd3afa94497 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-17T15:39:38,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117c98054f2dfde412580c0d9ae5ef198c4_a682ec5138f7084a1f2f9cd3afa94497 is 50, key is test_row_0/A:col10/1731857977709/Put/seqid=0 2024-11-17T15:39:38,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742402_1578 (size=12304) 2024-11-17T15:39:38,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(8855): Flush requested on a682ec5138f7084a1f2f9cd3afa94497 2024-11-17T15:39:38,846 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497. as already flushing 2024-11-17T15:39:38,881 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:38,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57374 deadline: 1731858038876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:38,882 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:38,882 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:38,882 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:38,883 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:38,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57310 deadline: 1731858038877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:38,884 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:38,884 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:38,884 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:38,884 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:38,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57332 deadline: 1731858038878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:38,884 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:38,884 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:38,884 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:38,884 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:38,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57288 deadline: 1731858038879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:38,884 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:38,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57380 deadline: 1731858038880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:38,884 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:38,884 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:38,884 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:38,885 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:38,885 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:38,885 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:38,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-17T15:39:38,994 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:38,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57374 deadline: 1731858038993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:38,995 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:38,995 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:38,995 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:38,995 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:38,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57332 deadline: 1731858038993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:38,995 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:38,995 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:38,995 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:38,997 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:38,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57310 deadline: 1731858038993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:38,997 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:38,997 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:38,997 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:38,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57380 deadline: 1731858038993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:38,997 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:38,997 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:38,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57288 deadline: 1731858038993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:38,997 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:38,997 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:38,998 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:38,998 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:38,998 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:38,998 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:39,092 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-17T15:39:39,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:39,141 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117c98054f2dfde412580c0d9ae5ef198c4_a682ec5138f7084a1f2f9cd3afa94497 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117c98054f2dfde412580c0d9ae5ef198c4_a682ec5138f7084a1f2f9cd3afa94497 2024-11-17T15:39:39,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/A/29924c3a7b0a4a06ba910061ade6f630, store: [table=TestAcidGuarantees family=A region=a682ec5138f7084a1f2f9cd3afa94497] 2024-11-17T15:39:39,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/A/29924c3a7b0a4a06ba910061ade6f630 is 175, key is test_row_0/A:col10/1731857977709/Put/seqid=0 2024-11-17T15:39:39,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742403_1579 (size=31105) 2024-11-17T15:39:39,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-17T15:39:39,208 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:39,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57374 deadline: 1731858039203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:39,208 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:39,208 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:39,208 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:39,208 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:39,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57332 deadline: 1731858039203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:39,209 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:39,209 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:39,209 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:39,209 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:39,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57310 deadline: 1731858039203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:39,209 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:39,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57380 deadline: 1731858039203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:39,209 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:39,209 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:39,209 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:39,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57288 deadline: 1731858039203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:39,209 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:39,209 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:39,209 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:39,209 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:39,210 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:39,210 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:39,210 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:39,516 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:39,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57374 deadline: 1731858039513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:39,516 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:39,516 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:39,516 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:39,516 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:39,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57332 deadline: 1731858039513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:39,516 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:39,516 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:39,516 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:39,516 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:39,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57310 deadline: 1731858039513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:39,517 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:39,517 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:39,517 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:39,517 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:39,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57380 deadline: 1731858039513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:39,517 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:39,517 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:39,517 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:39,517 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:39,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57288 deadline: 1731858039513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:39,517 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:39,517 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:39,517 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:39,552 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=208, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/A/29924c3a7b0a4a06ba910061ade6f630 2024-11-17T15:39:39,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/B/3d3fc79339fe4c45972bc0f078d64545 is 50, key is test_row_0/B:col10/1731857977709/Put/seqid=0 2024-11-17T15:39:39,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742404_1580 (size=12151) 2024-11-17T15:39:39,562 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/B/3d3fc79339fe4c45972bc0f078d64545 2024-11-17T15:39:39,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/C/0de620996731476fb19bcba0e55d3b17 is 50, key is test_row_0/C:col10/1731857977709/Put/seqid=0 2024-11-17T15:39:39,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742405_1581 (size=12151) 2024-11-17T15:39:39,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-17T15:39:39,975 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/C/0de620996731476fb19bcba0e55d3b17 2024-11-17T15:39:39,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/A/29924c3a7b0a4a06ba910061ade6f630 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/A/29924c3a7b0a4a06ba910061ade6f630 2024-11-17T15:39:39,981 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/A/29924c3a7b0a4a06ba910061ade6f630, entries=150, sequenceid=208, filesize=30.4 K 2024-11-17T15:39:39,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/B/3d3fc79339fe4c45972bc0f078d64545 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/B/3d3fc79339fe4c45972bc0f078d64545 2024-11-17T15:39:39,985 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/B/3d3fc79339fe4c45972bc0f078d64545, entries=150, sequenceid=208, filesize=11.9 K 2024-11-17T15:39:39,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/C/0de620996731476fb19bcba0e55d3b17 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/C/0de620996731476fb19bcba0e55d3b17 2024-11-17T15:39:39,988 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/C/0de620996731476fb19bcba0e55d3b17, entries=150, sequenceid=208, filesize=11.9 K 2024-11-17T15:39:39,989 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for a682ec5138f7084a1f2f9cd3afa94497 in 1261ms, sequenceid=208, compaction requested=true 2024-11-17T15:39:39,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2603): Flush status journal for a682ec5138f7084a1f2f9cd3afa94497: 2024-11-17T15:39:39,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497. 2024-11-17T15:39:39,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=161 2024-11-17T15:39:39,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.HMaster(4169): Remote procedure done, pid=161 2024-11-17T15:39:39,992 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=160 2024-11-17T15:39:39,992 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4140 sec 2024-11-17T15:39:39,994 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees in 1.4190 sec 2024-11-17T15:39:40,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(8855): Flush requested on a682ec5138f7084a1f2f9cd3afa94497 2024-11-17T15:39:40,027 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T15:39:40,027 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a682ec5138f7084a1f2f9cd3afa94497 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-17T15:39:40,035 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411175b0432fc31d14262a2f1f1e71f555c90_a682ec5138f7084a1f2f9cd3afa94497 is 50, key is test_row_0/A:col10/1731857978877/Put/seqid=0 2024-11-17T15:39:40,041 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:40,041 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:40,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57374 deadline: 1731858040036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:40,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57332 deadline: 1731858040036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:40,042 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:40,042 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:40,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57310 deadline: 1731858040037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:40,042 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:40,042 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:40,042 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:40,042 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:40,042 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:40,042 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:40,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57380 deadline: 1731858040037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:40,042 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:40,042 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:40,042 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:40,042 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:40,042 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:40,042 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:40,042 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:40,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57288 deadline: 1731858040037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:40,043 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:40,043 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:40,043 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:40,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742406_1582 (size=17284) 2024-11-17T15:39:40,154 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:40,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57332 deadline: 1731858040153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:40,155 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:40,155 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:40,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57310 deadline: 1731858040153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:40,155 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:40,155 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:40,155 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:40,155 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:40,155 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:40,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57374 deadline: 1731858040153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:40,155 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:40,155 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:40,156 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:40,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57380 deadline: 1731858040153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:40,156 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:40,156 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:40,156 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:40,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57288 deadline: 1731858040153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:40,156 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:40,156 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:40,156 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:40,156 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:40,156 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:40,156 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:40,367 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:40,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57332 deadline: 1731858040363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:40,367 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:40,367 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:40,367 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:40,367 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:40,367 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:40,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57310 deadline: 1731858040363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:40,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57374 deadline: 1731858040363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:40,367 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:40,368 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:40,368 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:40,368 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:40,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57380 deadline: 1731858040363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:40,368 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:40,368 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:40,368 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:40,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57288 deadline: 1731858040363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:40,368 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:40,368 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:40,368 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:40,368 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:40,368 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:40,368 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:40,368 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:40,444 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:40,448 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411175b0432fc31d14262a2f1f1e71f555c90_a682ec5138f7084a1f2f9cd3afa94497 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411175b0432fc31d14262a2f1f1e71f555c90_a682ec5138f7084a1f2f9cd3afa94497 2024-11-17T15:39:40,448 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/A/aa4407ce8639450c9af5eed2e3d568f6, store: [table=TestAcidGuarantees family=A region=a682ec5138f7084a1f2f9cd3afa94497] 2024-11-17T15:39:40,449 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/A/aa4407ce8639450c9af5eed2e3d568f6 is 175, key is test_row_0/A:col10/1731857978877/Put/seqid=0 2024-11-17T15:39:40,460 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742407_1583 (size=48389) 2024-11-17T15:39:40,676 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:40,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57332 deadline: 1731858040673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:40,677 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:40,677 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:40,677 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:40,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57310 deadline: 1731858040673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:40,677 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:40,677 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:40,677 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:40,677 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:40,677 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:40,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57374 deadline: 1731858040673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:40,677 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:40,677 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:40,677 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:40,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57380 deadline: 1731858040673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:40,677 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:40,677 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:40,678 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:40,678 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:40,678 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:40,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57288 deadline: 1731858040673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:40,678 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:40,678 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:40,678 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:40,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-17T15:39:40,723 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T15:39:40,723 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T15:39:40,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] procedure2.ProcedureExecutor(1139): Stored pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees 2024-11-17T15:39:40,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-17T15:39:40,725 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T15:39:40,726 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T15:39:40,726 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=163, ppid=162, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T15:39:40,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-17T15:39:40,861 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=235, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/A/aa4407ce8639450c9af5eed2e3d568f6 2024-11-17T15:39:40,867 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/B/d2ce283fe14d4c3985e788acf6d96bdf is 50, key is test_row_0/B:col10/1731857978877/Put/seqid=0 2024-11-17T15:39:40,878 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-17T15:39:40,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497. 2024-11-17T15:39:40,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497. as already flushing 2024-11-17T15:39:40,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497. 2024-11-17T15:39:40,878 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => a682ec5138f7084a1f2f9cd3afa94497, NAME => 'TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:39:40,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => a682ec5138f7084a1f2f9cd3afa94497, NAME => 'TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:39:40,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a682ec5138f7084a1f2f9cd3afa94497, NAME => 'TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a682ec5138f7084a1f2f9cd3afa94497, NAME => 'TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:39:40,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742408_1584 (size=12151) 2024-11-17T15:39:41,032 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-17T15:39:41,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497. 2024-11-17T15:39:41,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497. as already flushing 2024-11-17T15:39:41,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497. 2024-11-17T15:39:41,032 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => a682ec5138f7084a1f2f9cd3afa94497, NAME => 'TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:39:41,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => a682ec5138f7084a1f2f9cd3afa94497, NAME => 'TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:39:41,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a682ec5138f7084a1f2f9cd3afa94497, NAME => 'TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a682ec5138f7084a1f2f9cd3afa94497, NAME => 'TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:39:41,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-17T15:39:41,185 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-17T15:39:41,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497. 2024-11-17T15:39:41,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497. as already flushing 2024-11-17T15:39:41,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497. 2024-11-17T15:39:41,185 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => a682ec5138f7084a1f2f9cd3afa94497, NAME => 'TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:39:41,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => a682ec5138f7084a1f2f9cd3afa94497, NAME => 'TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:39:41,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a682ec5138f7084a1f2f9cd3afa94497, NAME => 'TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a682ec5138f7084a1f2f9cd3afa94497, NAME => 'TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:39:41,188 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:41,188 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:41,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57310 deadline: 1731858041183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:41,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57332 deadline: 1731858041183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:41,188 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:41,188 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:41,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57374 deadline: 1731858041184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:41,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57380 deadline: 1731858041184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:41,188 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:41,188 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:41,188 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:41,188 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:41,188 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:41,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57288 deadline: 1731858041184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:41,188 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:41,188 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:41,189 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:41,189 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:41,189 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:41,189 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:41,189 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:41,189 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:41,189 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:41,189 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:41,189 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:41,286 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/B/d2ce283fe14d4c3985e788acf6d96bdf 2024-11-17T15:39:41,297 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/C/738f2d5b6e084adcadf5c217b64796f6 is 50, key is test_row_0/C:col10/1731857978877/Put/seqid=0 2024-11-17T15:39:41,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742409_1585 (size=12151) 2024-11-17T15:39:41,337 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-17T15:39:41,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497. 2024-11-17T15:39:41,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497. as already flushing 2024-11-17T15:39:41,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497. 2024-11-17T15:39:41,337 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => a682ec5138f7084a1f2f9cd3afa94497, NAME => 'TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:39:41,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => a682ec5138f7084a1f2f9cd3afa94497, NAME => 'TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:39:41,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a682ec5138f7084a1f2f9cd3afa94497, NAME => 'TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a682ec5138f7084a1f2f9cd3afa94497, NAME => 'TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:39:41,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-17T15:39:41,490 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-17T15:39:41,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497. 2024-11-17T15:39:41,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497. as already flushing 2024-11-17T15:39:41,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497. 2024-11-17T15:39:41,490 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => a682ec5138f7084a1f2f9cd3afa94497, NAME => 'TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:39:41,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => a682ec5138f7084a1f2f9cd3afa94497, NAME => 'TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:39:41,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a682ec5138f7084a1f2f9cd3afa94497, NAME => 'TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a682ec5138f7084a1f2f9cd3afa94497, NAME => 'TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:39:41,643 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-17T15:39:41,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497. 2024-11-17T15:39:41,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497. as already flushing 2024-11-17T15:39:41,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497. 2024-11-17T15:39:41,643 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => a682ec5138f7084a1f2f9cd3afa94497, NAME => 'TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:39:41,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => a682ec5138f7084a1f2f9cd3afa94497, NAME => 'TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:39:41,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a682ec5138f7084a1f2f9cd3afa94497, NAME => 'TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a682ec5138f7084a1f2f9cd3afa94497, NAME => 'TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:39:41,701 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/C/738f2d5b6e084adcadf5c217b64796f6 2024-11-17T15:39:41,705 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/A/aa4407ce8639450c9af5eed2e3d568f6 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/A/aa4407ce8639450c9af5eed2e3d568f6 2024-11-17T15:39:41,708 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/A/aa4407ce8639450c9af5eed2e3d568f6, entries=250, sequenceid=235, filesize=47.3 K 2024-11-17T15:39:41,708 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/B/d2ce283fe14d4c3985e788acf6d96bdf as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/B/d2ce283fe14d4c3985e788acf6d96bdf 2024-11-17T15:39:41,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,712 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/B/d2ce283fe14d4c3985e788acf6d96bdf, entries=150, sequenceid=235, filesize=11.9 K 2024-11-17T15:39:41,712 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/C/738f2d5b6e084adcadf5c217b64796f6 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/C/738f2d5b6e084adcadf5c217b64796f6 2024-11-17T15:39:41,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,716 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/C/738f2d5b6e084adcadf5c217b64796f6, entries=150, sequenceid=235, filesize=11.9 K 2024-11-17T15:39:41,717 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for a682ec5138f7084a1f2f9cd3afa94497 in 1690ms, sequenceid=235, compaction requested=true 2024-11-17T15:39:41,717 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a682ec5138f7084a1f2f9cd3afa94497: 2024-11-17T15:39:41,717 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a682ec5138f7084a1f2f9cd3afa94497:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T15:39:41,717 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T15:39:41,717 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a682ec5138f7084a1f2f9cd3afa94497:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T15:39:41,717 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T15:39:41,717 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a682ec5138f7084a1f2f9cd3afa94497:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T15:39:41,717 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T15:39:41,717 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-11-17T15:39:41,717 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-11-17T15:39:41,718 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 73080 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-11-17T15:39:41,718 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 212732 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-11-17T15:39:41,718 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1541): a682ec5138f7084a1f2f9cd3afa94497/B is initiating minor compaction (all files) 2024-11-17T15:39:41,718 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1541): a682ec5138f7084a1f2f9cd3afa94497/A is initiating minor compaction (all files) 2024-11-17T15:39:41,718 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a682ec5138f7084a1f2f9cd3afa94497/B in TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497. 2024-11-17T15:39:41,718 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a682ec5138f7084a1f2f9cd3afa94497/A in TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497. 2024-11-17T15:39:41,718 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/B/5f3f51f9e48a49d0bfadc9663cec248a, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/B/376b2eec641f4db18b5483d68ec80df1, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/B/edcccf8a10c041d28b11d339d20d70c4, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/B/9e065d0e15f44fec9daffa63c10ccc7d, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/B/3d3fc79339fe4c45972bc0f078d64545, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/B/d2ce283fe14d4c3985e788acf6d96bdf] into tmpdir=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp, totalSize=71.4 K 2024-11-17T15:39:41,718 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/A/fe806eb1dbd44354ba9f546e943ba3f7, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/A/3604334c372a467ab844d06f04087498, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/A/415ab553c1a54754be54ca8433b84166, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/A/dc16ad2d35744a4095376e009088dabc, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/A/29924c3a7b0a4a06ba910061ade6f630, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/A/aa4407ce8639450c9af5eed2e3d568f6] into tmpdir=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp, totalSize=207.7 K 2024-11-17T15:39:41,719 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=10 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497. 2024-11-17T15:39:41,719 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497. files: [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/A/fe806eb1dbd44354ba9f546e943ba3f7, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/A/3604334c372a467ab844d06f04087498, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/A/415ab553c1a54754be54ca8433b84166, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/A/dc16ad2d35744a4095376e009088dabc, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/A/29924c3a7b0a4a06ba910061ade6f630, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/A/aa4407ce8639450c9af5eed2e3d568f6] 2024-11-17T15:39:41,719 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting 5f3f51f9e48a49d0bfadc9663cec248a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731857973874 2024-11-17T15:39:41,719 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting fe806eb1dbd44354ba9f546e943ba3f7, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731857973874 2024-11-17T15:39:41,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,719 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting 376b2eec641f4db18b5483d68ec80df1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731857975054 2024-11-17T15:39:41,720 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting edcccf8a10c041d28b11d339d20d70c4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1731857975179 2024-11-17T15:39:41,720 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3604334c372a467ab844d06f04087498, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731857975053 2024-11-17T15:39:41,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,720 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 415ab553c1a54754be54ca8433b84166, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1731857975179 2024-11-17T15:39:41,720 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting 9e065d0e15f44fec9daffa63c10ccc7d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731857975530 2024-11-17T15:39:41,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,720 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting dc16ad2d35744a4095376e009088dabc, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731857975530 2024-11-17T15:39:41,720 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting 3d3fc79339fe4c45972bc0f078d64545, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1731857977707 2024-11-17T15:39:41,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,721 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 29924c3a7b0a4a06ba910061ade6f630, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1731857977707 2024-11-17T15:39:41,721 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting d2ce283fe14d4c3985e788acf6d96bdf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1731857978877 2024-11-17T15:39:41,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,721 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting aa4407ce8639450c9af5eed2e3d568f6, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1731857978877 2024-11-17T15:39:41,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,733 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a682ec5138f7084a1f2f9cd3afa94497#B#compaction#491 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T15:39:41,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,733 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/B/57e89d6207814cf3858029798a607c9c is 50, key is test_row_0/B:col10/1731857978877/Put/seqid=0 2024-11-17T15:39:41,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,734 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a682ec5138f7084a1f2f9cd3afa94497] 2024-11-17T15:39:41,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,737 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411176f0517254c7b4d8593e2de9e7c2266be_a682ec5138f7084a1f2f9cd3afa94497 store=[table=TestAcidGuarantees family=A region=a682ec5138f7084a1f2f9cd3afa94497] 2024-11-17T15:39:41,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,739 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411176f0517254c7b4d8593e2de9e7c2266be_a682ec5138f7084a1f2f9cd3afa94497, store=[table=TestAcidGuarantees family=A region=a682ec5138f7084a1f2f9cd3afa94497] 2024-11-17T15:39:41,739 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411176f0517254c7b4d8593e2de9e7c2266be_a682ec5138f7084a1f2f9cd3afa94497 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a682ec5138f7084a1f2f9cd3afa94497] 2024-11-17T15:39:41,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742410_1586 (size=12629) 2024-11-17T15:39:41,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742411_1587 (size=4469) 2024-11-17T15:39:41,749 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a682ec5138f7084a1f2f9cd3afa94497#A#compaction#492 average throughput is 1.63 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T15:39:41,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,750 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/A/ebf05454530f40768b5a11e90355521b is 175, key is test_row_0/A:col10/1731857978877/Put/seqid=0 2024-11-17T15:39:41,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742412_1588 (size=31583) 2024-11-17T15:39:41,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,780 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/A/ebf05454530f40768b5a11e90355521b as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/A/ebf05454530f40768b5a11e90355521b 2024-11-17T15:39:41,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,785 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in a682ec5138f7084a1f2f9cd3afa94497/A of a682ec5138f7084a1f2f9cd3afa94497 into ebf05454530f40768b5a11e90355521b(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T15:39:41,785 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a682ec5138f7084a1f2f9cd3afa94497: 2024-11-17T15:39:41,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,785 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., storeName=a682ec5138f7084a1f2f9cd3afa94497/A, priority=10, startTime=1731857981717; duration=0sec 2024-11-17T15:39:41,785 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T15:39:41,785 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a682ec5138f7084a1f2f9cd3afa94497:A 2024-11-17T15:39:41,785 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-11-17T15:39:41,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,787 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 73080 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-11-17T15:39:41,787 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1541): a682ec5138f7084a1f2f9cd3afa94497/C is initiating minor compaction (all files) 2024-11-17T15:39:41,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,787 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a682ec5138f7084a1f2f9cd3afa94497/C in TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497. 2024-11-17T15:39:41,787 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/C/cad4210b0b6d4656b6ec08af997136b5, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/C/21ab8f1c786c40b6997dc6fa2a1cf6a4, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/C/cd225f855b884a6cbc022d4a0a571f55, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/C/2f12e08c211a4581a7d222a6bc31524d, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/C/0de620996731476fb19bcba0e55d3b17, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/C/738f2d5b6e084adcadf5c217b64796f6] into tmpdir=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp, totalSize=71.4 K 2024-11-17T15:39:41,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,787 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting cad4210b0b6d4656b6ec08af997136b5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731857973874 2024-11-17T15:39:41,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,788 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 21ab8f1c786c40b6997dc6fa2a1cf6a4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731857975054 2024-11-17T15:39:41,788 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting cd225f855b884a6cbc022d4a0a571f55, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1731857975179 2024-11-17T15:39:41,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,788 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2f12e08c211a4581a7d222a6bc31524d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731857975530 2024-11-17T15:39:41,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,789 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0de620996731476fb19bcba0e55d3b17, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1731857977707 2024-11-17T15:39:41,789 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 738f2d5b6e084adcadf5c217b64796f6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1731857978877 2024-11-17T15:39:41,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,796 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-17T15:39:41,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497. 2024-11-17T15:39:41,796 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2902): Flushing a682ec5138f7084a1f2f9cd3afa94497 3/3 column families, dataSize=40.25 KB heapSize=106.22 KB 2024-11-17T15:39:41,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,800 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a682ec5138f7084a1f2f9cd3afa94497#C#compaction#493 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T15:39:41,800 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/C/c3ebde68060e41ce85ee6bb4cec4376f is 50, key is test_row_0/C:col10/1731857978877/Put/seqid=0 2024-11-17T15:39:41,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742413_1589 (size=12629) 2024-11-17T15:39:41,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,822 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/C/c3ebde68060e41ce85ee6bb4cec4376f as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/C/c3ebde68060e41ce85ee6bb4cec4376f 2024-11-17T15:39:41,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,826 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in a682ec5138f7084a1f2f9cd3afa94497/C of a682ec5138f7084a1f2f9cd3afa94497 into c3ebde68060e41ce85ee6bb4cec4376f(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T15:39:41,826 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a682ec5138f7084a1f2f9cd3afa94497: 2024-11-17T15:39:41,826 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., storeName=a682ec5138f7084a1f2f9cd3afa94497/C, priority=10, startTime=1731857981717; duration=0sec 2024-11-17T15:39:41,826 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T15:39:41,826 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a682ec5138f7084a1f2f9cd3afa94497:C 2024-11-17T15:39:41,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111798fbd0ace52f4f7cb0a62bb42d62ad69_a682ec5138f7084a1f2f9cd3afa94497 is 50, key is test_row_0/A:col10/1731857980035/Put/seqid=0 2024-11-17T15:39:41,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,859 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742414_1590 (size=9814) 2024-11-17T15:39:41,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-17T15:39:41,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:41,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,146 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/B/57e89d6207814cf3858029798a607c9c as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/B/57e89d6207814cf3858029798a607c9c 2024-11-17T15:39:42,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,151 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in a682ec5138f7084a1f2f9cd3afa94497/B of a682ec5138f7084a1f2f9cd3afa94497 into 57e89d6207814cf3858029798a607c9c(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T15:39:42,151 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a682ec5138f7084a1f2f9cd3afa94497: 2024-11-17T15:39:42,151 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., storeName=a682ec5138f7084a1f2f9cd3afa94497/B, priority=10, startTime=1731857981717; duration=0sec 2024-11-17T15:39:42,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,151 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T15:39:42,152 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a682ec5138f7084a1f2f9cd3afa94497:B 2024-11-17T15:39:42,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(8855): Flush requested on a682ec5138f7084a1f2f9cd3afa94497 2024-11-17T15:39:42,201 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497. as already flushing 2024-11-17T15:39:42,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,237 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:42,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57374 deadline: 1731858042233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:42,237 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:42,237 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:42,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57332 deadline: 1731858042233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:42,237 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:42,237 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:42,238 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:42,238 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:42,238 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:42,238 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:42,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57288 deadline: 1731858042233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:42,238 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:42,238 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:42,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57380 deadline: 1731858042234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:42,238 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:42,238 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:42,238 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:42,238 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:42,238 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:42,238 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:42,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57310 deadline: 1731858042236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:42,239 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:42,239 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:42,239 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:42,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:42,263 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111798fbd0ace52f4f7cb0a62bb42d62ad69_a682ec5138f7084a1f2f9cd3afa94497 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111798fbd0ace52f4f7cb0a62bb42d62ad69_a682ec5138f7084a1f2f9cd3afa94497 2024-11-17T15:39:42,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/A/2f81797602ce4532803ca5dced176683, store: [table=TestAcidGuarantees family=A region=a682ec5138f7084a1f2f9cd3afa94497] 2024-11-17T15:39:42,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/A/2f81797602ce4532803ca5dced176683 is 175, key is test_row_0/A:col10/1731857980035/Put/seqid=0 2024-11-17T15:39:42,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742415_1591 (size=22461) 2024-11-17T15:39:42,344 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:42,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57374 deadline: 1731858042343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:42,345 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:42,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57332 deadline: 1731858042343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:42,345 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:42,345 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:42,345 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:42,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57288 deadline: 1731858042343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:42,345 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:42,345 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:42,345 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:42,345 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:42,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57380 deadline: 1731858042343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:42,345 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:42,345 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:42,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57310 deadline: 1731858042343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:42,346 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:42,346 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:42,346 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:42,346 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:42,346 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:42,346 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:42,346 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:42,346 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:42,346 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:42,556 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:42,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57332 deadline: 1731858042553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:42,557 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:42,557 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:42,557 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:42,557 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:42,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57310 deadline: 1731858042554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:42,557 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:42,557 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:42,557 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:42,557 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:42,557 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:42,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57374 deadline: 1731858042554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:42,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57288 deadline: 1731858042554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:42,558 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:42,558 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:42,558 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:42,558 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:42,558 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:42,558 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:42,558 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:42,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57380 deadline: 1731858042554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:42,558 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:42,558 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:42,558 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:42,671 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=245, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/A/2f81797602ce4532803ca5dced176683 2024-11-17T15:39:42,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/B/156f3d7037ae4101b961756214768686 is 50, key is test_row_0/B:col10/1731857980035/Put/seqid=0 2024-11-17T15:39:42,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742416_1592 (size=9757) 2024-11-17T15:39:42,865 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:42,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57332 deadline: 1731858042863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:42,865 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:42,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57374 deadline: 1731858042863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:42,865 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:42,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57288 deadline: 1731858042863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:42,866 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:42,866 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:42,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57310 deadline: 1731858042864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:42,866 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:42,866 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:42,866 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:42,866 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:42,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57380 deadline: 1731858042864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:42,866 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:42,866 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:42,867 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:42,867 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:42,867 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:42,867 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:42,867 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:42,867 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:42,869 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:42,869 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:42,869 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:42,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-17T15:39:43,099 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/B/156f3d7037ae4101b961756214768686 2024-11-17T15:39:43,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/C/b1f3f0b436ac4456a2ee333f58814330 is 50, key is test_row_0/C:col10/1731857980035/Put/seqid=0 2024-11-17T15:39:43,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742417_1593 (size=9757) 2024-11-17T15:39:43,132 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/C/b1f3f0b436ac4456a2ee333f58814330 2024-11-17T15:39:43,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/A/2f81797602ce4532803ca5dced176683 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/A/2f81797602ce4532803ca5dced176683 2024-11-17T15:39:43,154 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/A/2f81797602ce4532803ca5dced176683, entries=100, sequenceid=245, filesize=21.9 K 2024-11-17T15:39:43,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/B/156f3d7037ae4101b961756214768686 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/B/156f3d7037ae4101b961756214768686 2024-11-17T15:39:43,159 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/B/156f3d7037ae4101b961756214768686, entries=100, sequenceid=245, filesize=9.5 K 2024-11-17T15:39:43,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/C/b1f3f0b436ac4456a2ee333f58814330 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/C/b1f3f0b436ac4456a2ee333f58814330 2024-11-17T15:39:43,164 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/C/b1f3f0b436ac4456a2ee333f58814330, entries=100, sequenceid=245, filesize=9.5 K 2024-11-17T15:39:43,165 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.17 KB/108720, currentSize=161.02 KB/164880 for a682ec5138f7084a1f2f9cd3afa94497 in 1369ms, sequenceid=245, compaction requested=false 2024-11-17T15:39:43,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2603): Flush status journal for a682ec5138f7084a1f2f9cd3afa94497: 2024-11-17T15:39:43,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497. 2024-11-17T15:39:43,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=163 2024-11-17T15:39:43,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.HMaster(4169): Remote procedure done, pid=163 2024-11-17T15:39:43,169 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=163, resume processing ppid=162 2024-11-17T15:39:43,169 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=163, ppid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4410 sec 2024-11-17T15:39:43,173 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees in 2.4480 sec 2024-11-17T15:39:43,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(8855): Flush requested on a682ec5138f7084a1f2f9cd3afa94497 2024-11-17T15:39:43,377 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T15:39:43,377 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a682ec5138f7084a1f2f9cd3afa94497 3/3 column families, dataSize=167.72 KB heapSize=440.20 KB 2024-11-17T15:39:43,387 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:43,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57374 deadline: 1731858043379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:43,387 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:43,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57288 deadline: 1731858043383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:43,387 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:43,387 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:43,387 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:43,387 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:43,387 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:43,387 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:43,390 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:43,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57332 deadline: 1731858043385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:43,390 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:43,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57310 deadline: 1731858043387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:43,391 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:43,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57380 deadline: 1731858043387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:43,391 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:43,391 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:43,391 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:43,391 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:43,391 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:43,391 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:43,392 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:43,392 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:43,392 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:43,401 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117da370652fdf347a496c9928e86863022_a682ec5138f7084a1f2f9cd3afa94497 is 50, key is test_row_0/A:col10/1731857982235/Put/seqid=0 2024-11-17T15:39:43,422 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742418_1594 (size=14994) 2024-11-17T15:39:43,423 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,428 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117da370652fdf347a496c9928e86863022_a682ec5138f7084a1f2f9cd3afa94497 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117da370652fdf347a496c9928e86863022_a682ec5138f7084a1f2f9cd3afa94497 2024-11-17T15:39:43,429 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/A/876d0cd83fce43b6a195dc0ed4571d03, store: [table=TestAcidGuarantees family=A region=a682ec5138f7084a1f2f9cd3afa94497] 2024-11-17T15:39:43,429 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/A/876d0cd83fce43b6a195dc0ed4571d03 is 175, key is test_row_0/A:col10/1731857982235/Put/seqid=0 2024-11-17T15:39:43,444 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742419_1595 (size=39949) 2024-11-17T15:39:43,445 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=278, memsize=62.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/A/876d0cd83fce43b6a195dc0ed4571d03 2024-11-17T15:39:43,455 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/B/c84029d3dbf04150874356bfbdc30517 is 50, key is test_row_0/B:col10/1731857982235/Put/seqid=0 2024-11-17T15:39:43,472 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742420_1596 (size=12301) 2024-11-17T15:39:43,473 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=62.62 KB at sequenceid=278 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/B/c84029d3dbf04150874356bfbdc30517 2024-11-17T15:39:43,486 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/C/98f53ddc189e4c038be7d19425785d5d is 50, key is test_row_0/C:col10/1731857982235/Put/seqid=0 2024-11-17T15:39:43,495 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:43,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57374 deadline: 1731858043494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:43,495 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:43,495 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:43,495 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:43,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57332 deadline: 1731858043494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:43,495 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:43,496 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:43,496 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:43,496 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:43,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57380 deadline: 1731858043494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:43,496 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:43,496 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:43,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57310 deadline: 1731858043494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:43,496 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:43,496 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:43,496 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:43,497 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:43,497 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:43,497 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:43,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742421_1597 (size=12301) 2024-11-17T15:39:43,504 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=62.62 KB at sequenceid=278 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/C/98f53ddc189e4c038be7d19425785d5d 2024-11-17T15:39:43,510 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/A/876d0cd83fce43b6a195dc0ed4571d03 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/A/876d0cd83fce43b6a195dc0ed4571d03 2024-11-17T15:39:43,515 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/A/876d0cd83fce43b6a195dc0ed4571d03, entries=200, sequenceid=278, filesize=39.0 K 2024-11-17T15:39:43,516 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/B/c84029d3dbf04150874356bfbdc30517 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/B/c84029d3dbf04150874356bfbdc30517 2024-11-17T15:39:43,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,520 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/B/c84029d3dbf04150874356bfbdc30517, entries=150, sequenceid=278, filesize=12.0 K 2024-11-17T15:39:43,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,521 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/C/98f53ddc189e4c038be7d19425785d5d as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/C/98f53ddc189e4c038be7d19425785d5d 2024-11-17T15:39:43,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,525 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/C/98f53ddc189e4c038be7d19425785d5d, entries=150, sequenceid=278, filesize=12.0 K 2024-11-17T15:39:43,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,526 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~187.85 KB/192360, heapSize ~492.89 KB/504720, currentSize=13.42 KB/13740 for a682ec5138f7084a1f2f9cd3afa94497 in 149ms, sequenceid=278, compaction requested=true 2024-11-17T15:39:43,527 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a682ec5138f7084a1f2f9cd3afa94497: 2024-11-17T15:39:43,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,527 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T15:39:43,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,527 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a682ec5138f7084a1f2f9cd3afa94497:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T15:39:43,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,527 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T15:39:43,527 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a682ec5138f7084a1f2f9cd3afa94497:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T15:39:43,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,527 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T15:39:43,527 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T15:39:43,527 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a682ec5138f7084a1f2f9cd3afa94497:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T15:39:43,527 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T15:39:43,528 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93993 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T15:39:43,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,528 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1541): a682ec5138f7084a1f2f9cd3afa94497/A is initiating minor compaction (all files) 2024-11-17T15:39:43,528 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a682ec5138f7084a1f2f9cd3afa94497/A in TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497. 2024-11-17T15:39:43,528 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/A/ebf05454530f40768b5a11e90355521b, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/A/2f81797602ce4532803ca5dced176683, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/A/876d0cd83fce43b6a195dc0ed4571d03] into tmpdir=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp, totalSize=91.8 K 2024-11-17T15:39:43,528 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497. 2024-11-17T15:39:43,528 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497. files: [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/A/ebf05454530f40768b5a11e90355521b, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/A/2f81797602ce4532803ca5dced176683, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/A/876d0cd83fce43b6a195dc0ed4571d03] 2024-11-17T15:39:43,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,529 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting ebf05454530f40768b5a11e90355521b, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1731857978877 2024-11-17T15:39:43,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,529 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T15:39:43,529 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2f81797602ce4532803ca5dced176683, keycount=100, bloomtype=ROW, size=21.9 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1731857980035 2024-11-17T15:39:43,530 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1541): a682ec5138f7084a1f2f9cd3afa94497/B is initiating minor compaction (all files) 2024-11-17T15:39:43,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,530 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a682ec5138f7084a1f2f9cd3afa94497/B in TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497. 2024-11-17T15:39:43,530 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/B/57e89d6207814cf3858029798a607c9c, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/B/156f3d7037ae4101b961756214768686, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/B/c84029d3dbf04150874356bfbdc30517] into tmpdir=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp, totalSize=33.9 K 2024-11-17T15:39:43,530 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 876d0cd83fce43b6a195dc0ed4571d03, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1731857982235 2024-11-17T15:39:43,530 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting 57e89d6207814cf3858029798a607c9c, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1731857978877 2024-11-17T15:39:43,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,531 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting 156f3d7037ae4101b961756214768686, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1731857980035 2024-11-17T15:39:43,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,531 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting c84029d3dbf04150874356bfbdc30517, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1731857982235 2024-11-17T15:39:43,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,546 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a682ec5138f7084a1f2f9cd3afa94497] 2024-11-17T15:39:43,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,554 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a682ec5138f7084a1f2f9cd3afa94497#B#compaction#501 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T15:39:43,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,555 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/B/08d112961f57431b8285e2d257fa1ace is 50, key is test_row_0/B:col10/1731857982235/Put/seqid=0 2024-11-17T15:39:43,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,557 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241117d6a6ec2831864fa4851b237e83a0bafd_a682ec5138f7084a1f2f9cd3afa94497 store=[table=TestAcidGuarantees family=A region=a682ec5138f7084a1f2f9cd3afa94497] 2024-11-17T15:39:43,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,558 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241117d6a6ec2831864fa4851b237e83a0bafd_a682ec5138f7084a1f2f9cd3afa94497, store=[table=TestAcidGuarantees family=A region=a682ec5138f7084a1f2f9cd3afa94497] 2024-11-17T15:39:43,558 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117d6a6ec2831864fa4851b237e83a0bafd_a682ec5138f7084a1f2f9cd3afa94497 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a682ec5138f7084a1f2f9cd3afa94497] 2024-11-17T15:39:43,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742422_1598 (size=12881) 2024-11-17T15:39:43,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,591 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/B/08d112961f57431b8285e2d257fa1ace as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/B/08d112961f57431b8285e2d257fa1ace 2024-11-17T15:39:43,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,597 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a682ec5138f7084a1f2f9cd3afa94497/B of a682ec5138f7084a1f2f9cd3afa94497 into 08d112961f57431b8285e2d257fa1ace(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T15:39:43,597 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a682ec5138f7084a1f2f9cd3afa94497: 2024-11-17T15:39:43,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,597 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., storeName=a682ec5138f7084a1f2f9cd3afa94497/B, priority=13, startTime=1731857983527; duration=0sec 2024-11-17T15:39:43,597 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T15:39:43,597 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a682ec5138f7084a1f2f9cd3afa94497:B 2024-11-17T15:39:43,597 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T15:39:43,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,598 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T15:39:43,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,598 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1541): a682ec5138f7084a1f2f9cd3afa94497/C is initiating minor compaction (all files) 2024-11-17T15:39:43,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,598 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a682ec5138f7084a1f2f9cd3afa94497/C in TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497. 2024-11-17T15:39:43,598 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/C/c3ebde68060e41ce85ee6bb4cec4376f, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/C/b1f3f0b436ac4456a2ee333f58814330, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/C/98f53ddc189e4c038be7d19425785d5d] into tmpdir=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp, totalSize=33.9 K 2024-11-17T15:39:43,599 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting c3ebde68060e41ce85ee6bb4cec4376f, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1731857978877 2024-11-17T15:39:43,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,599 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting b1f3f0b436ac4456a2ee333f58814330, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1731857980035 2024-11-17T15:39:43,599 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting 98f53ddc189e4c038be7d19425785d5d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1731857982235 2024-11-17T15:39:43,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742423_1599 (size=4469) 2024-11-17T15:39:43,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,607 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a682ec5138f7084a1f2f9cd3afa94497#C#compaction#502 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T15:39:43,608 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/C/135dff1cf42241348acb4c354a76866c is 50, key is test_row_0/C:col10/1731857982235/Put/seqid=0 2024-11-17T15:39:43,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,610 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a682ec5138f7084a1f2f9cd3afa94497#A#compaction#500 average throughput is 0.39 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T15:39:43,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,610 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/A/ec79ac6bdc6643f3a56d28cbceee8373 is 175, key is test_row_0/A:col10/1731857982235/Put/seqid=0 2024-11-17T15:39:43,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742424_1600 (size=12881) 2024-11-17T15:39:43,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,641 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/C/135dff1cf42241348acb4c354a76866c as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/C/135dff1cf42241348acb4c354a76866c 2024-11-17T15:39:43,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742425_1601 (size=31835) 2024-11-17T15:39:43,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,650 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a682ec5138f7084a1f2f9cd3afa94497/C of a682ec5138f7084a1f2f9cd3afa94497 into 135dff1cf42241348acb4c354a76866c(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T15:39:43,650 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a682ec5138f7084a1f2f9cd3afa94497: 2024-11-17T15:39:43,650 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., storeName=a682ec5138f7084a1f2f9cd3afa94497/C, priority=13, startTime=1731857983527; duration=0sec 2024-11-17T15:39:43,651 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T15:39:43,651 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a682ec5138f7084a1f2f9cd3afa94497:C 2024-11-17T15:39:43,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,663 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/A/ec79ac6bdc6643f3a56d28cbceee8373 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/A/ec79ac6bdc6643f3a56d28cbceee8373 2024-11-17T15:39:43,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,669 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a682ec5138f7084a1f2f9cd3afa94497/A of a682ec5138f7084a1f2f9cd3afa94497 into ec79ac6bdc6643f3a56d28cbceee8373(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T15:39:43,669 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a682ec5138f7084a1f2f9cd3afa94497: 2024-11-17T15:39:43,669 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., storeName=a682ec5138f7084a1f2f9cd3afa94497/A, priority=13, startTime=1731857983527; duration=0sec 2024-11-17T15:39:43,669 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T15:39:43,669 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a682ec5138f7084a1f2f9cd3afa94497:A 2024-11-17T15:39:43,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(8855): Flush requested on a682ec5138f7084a1f2f9cd3afa94497 2024-11-17T15:39:43,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,720 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T15:39:43,720 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a682ec5138f7084a1f2f9cd3afa94497 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-17T15:39:43,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,739 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411170bf553063f0e40f2b4e1fd3cd042c0a7_a682ec5138f7084a1f2f9cd3afa94497 is 50, key is test_row_0/A:col10/1731857983718/Put/seqid=0 2024-11-17T15:39:43,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,762 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:43,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57374 deadline: 1731858043756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:43,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742427_1603 (size=30238) 2024-11-17T15:39:43,762 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:43,762 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:43,763 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:43,763 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:43,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57310 deadline: 1731858043757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:43,763 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,764 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:43,764 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:43,764 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:43,764 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:43,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57380 deadline: 1731858043759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:43,765 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:43,765 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:43,765 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:43,767 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:43,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57332 deadline: 1731858043763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:43,767 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411170bf553063f0e40f2b4e1fd3cd042c0a7_a682ec5138f7084a1f2f9cd3afa94497 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411170bf553063f0e40f2b4e1fd3cd042c0a7_a682ec5138f7084a1f2f9cd3afa94497 2024-11-17T15:39:43,768 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:43,768 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:43,768 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:43,768 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/A/67a8fcefd62347529545281f4f125f5b, store: [table=TestAcidGuarantees family=A region=a682ec5138f7084a1f2f9cd3afa94497] 2024-11-17T15:39:43,769 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/A/67a8fcefd62347529545281f4f125f5b is 175, key is test_row_0/A:col10/1731857983718/Put/seqid=0 2024-11-17T15:39:43,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742426_1602 (size=92179) 2024-11-17T15:39:43,771 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=294, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/A/67a8fcefd62347529545281f4f125f5b 2024-11-17T15:39:43,777 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/B/acf9528f40ab418ca11bc1f10ced4af8 is 50, key is test_row_0/B:col10/1731857983718/Put/seqid=0 2024-11-17T15:39:43,780 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742428_1604 (size=12301) 2024-11-17T15:39:43,780 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=294 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/B/acf9528f40ab418ca11bc1f10ced4af8 2024-11-17T15:39:43,794 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/C/34c973e46037411f8d07d43dbb49b98d is 50, key is test_row_0/C:col10/1731857983718/Put/seqid=0 2024-11-17T15:39:43,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742429_1605 (size=12301) 2024-11-17T15:39:43,827 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=294 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/C/34c973e46037411f8d07d43dbb49b98d 2024-11-17T15:39:43,831 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/A/67a8fcefd62347529545281f4f125f5b as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/A/67a8fcefd62347529545281f4f125f5b 2024-11-17T15:39:43,839 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/A/67a8fcefd62347529545281f4f125f5b, entries=500, sequenceid=294, filesize=90.0 K 2024-11-17T15:39:43,839 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/B/acf9528f40ab418ca11bc1f10ced4af8 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/B/acf9528f40ab418ca11bc1f10ced4af8 2024-11-17T15:39:43,847 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/B/acf9528f40ab418ca11bc1f10ced4af8, entries=150, sequenceid=294, filesize=12.0 K 2024-11-17T15:39:43,848 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/C/34c973e46037411f8d07d43dbb49b98d as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/C/34c973e46037411f8d07d43dbb49b98d 2024-11-17T15:39:43,852 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/C/34c973e46037411f8d07d43dbb49b98d, entries=150, sequenceid=294, filesize=12.0 K 2024-11-17T15:39:43,854 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for a682ec5138f7084a1f2f9cd3afa94497 in 134ms, sequenceid=294, compaction requested=false 2024-11-17T15:39:43,854 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a682ec5138f7084a1f2f9cd3afa94497: 2024-11-17T15:39:43,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(8855): Flush requested on a682ec5138f7084a1f2f9cd3afa94497 2024-11-17T15:39:43,878 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T15:39:43,878 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a682ec5138f7084a1f2f9cd3afa94497 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-17T15:39:43,893 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:43,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57380 deadline: 1731858043889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:43,894 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:43,894 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:43,894 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:43,897 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:43,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57332 deadline: 1731858043892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:43,898 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:43,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57310 deadline: 1731858043892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:43,898 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:43,898 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:43,898 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:43,898 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:43,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57374 deadline: 1731858043894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:43,899 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:43,899 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:43,899 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:43,899 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:43,899 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:43,899 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:43,913 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411177fbed3a2d48d4655bac54573441b7fa9_a682ec5138f7084a1f2f9cd3afa94497 is 50, key is test_row_0/A:col10/1731857983757/Put/seqid=0 2024-11-17T15:39:43,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742430_1606 (size=17534) 2024-11-17T15:39:43,965 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:43,970 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411177fbed3a2d48d4655bac54573441b7fa9_a682ec5138f7084a1f2f9cd3afa94497 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411177fbed3a2d48d4655bac54573441b7fa9_a682ec5138f7084a1f2f9cd3afa94497 2024-11-17T15:39:43,971 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/A/65e0522a067240c1bf0553ec1bf65235, store: [table=TestAcidGuarantees family=A region=a682ec5138f7084a1f2f9cd3afa94497] 2024-11-17T15:39:43,972 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/A/65e0522a067240c1bf0553ec1bf65235 is 175, key is test_row_0/A:col10/1731857983757/Put/seqid=0 2024-11-17T15:39:44,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742431_1607 (size=48639) 2024-11-17T15:39:44,007 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:44,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57374 deadline: 1731858044003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:44,007 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:44,007 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:44,008 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:44,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57332 deadline: 1731858044003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:44,008 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:44,008 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:44,008 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:44,008 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:44,008 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:44,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57310 deadline: 1731858044003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:44,008 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:44,008 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:44,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57380 deadline: 1731858044003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:44,008 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:44,008 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:44,009 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:44,009 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:44,009 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:44,215 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:44,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57374 deadline: 1731858044213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:44,216 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:44,216 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:44,216 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:44,216 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:44,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57332 deadline: 1731858044213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:44,217 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:44,217 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:44,217 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:44,217 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:44,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57380 deadline: 1731858044213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:44,217 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:44,217 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:44,217 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:44,217 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:44,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57310 deadline: 1731858044213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:44,217 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:44,217 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:44,217 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:44,403 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=321, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/A/65e0522a067240c1bf0553ec1bf65235 2024-11-17T15:39:44,406 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:44,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57288 deadline: 1731858044403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:44,407 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 7a780d55532c,32955,1731857800648 for 'test_row_0' in a682ec5138f7084a1f2f9cd3afa94497 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2174 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T15:39:44,408 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:44,408 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:44,408 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:44,419 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/B/df4115e30f234f84a9eb1282e785e0fb is 50, key is test_row_0/B:col10/1731857983757/Put/seqid=0 2024-11-17T15:39:44,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742432_1608 (size=12301) 2024-11-17T15:39:44,525 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:44,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57374 deadline: 1731858044523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:44,525 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:44,525 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:44,526 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:44,526 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:44,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57332 deadline: 1731858044523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:44,526 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:44,526 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:44,526 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:44,526 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:44,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57380 deadline: 1731858044523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:44,526 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:44,526 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:44,526 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:44,526 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:44,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57310 deadline: 1731858044523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:44,527 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:44,527 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:44,527 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:44,830 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=321 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/B/df4115e30f234f84a9eb1282e785e0fb 2024-11-17T15:39:44,836 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/C/c7954d30fec540a885f670a7821e9690 is 50, key is test_row_0/C:col10/1731857983757/Put/seqid=0 2024-11-17T15:39:44,839 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742433_1609 (size=12301) 2024-11-17T15:39:44,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-17T15:39:44,904 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T15:39:44,904 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T15:39:44,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] procedure2.ProcedureExecutor(1139): Stored pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees 2024-11-17T15:39:44,906 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T15:39:44,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-17T15:39:44,907 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T15:39:44,907 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=165, ppid=164, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T15:39:45,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-17T15:39:45,034 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:45,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57374 deadline: 1731858045033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:45,034 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:45,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57332 deadline: 1731858045034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:45,035 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:45,035 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:45,035 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:45,035 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:45,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57310 deadline: 1731858045034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:45,035 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:45,035 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:45,035 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:45,035 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:45,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57380 deadline: 1731858045034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:45,035 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:45,035 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:45,035 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:45,035 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:45,035 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:45,035 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:45,059 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-17T15:39:45,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497. 2024-11-17T15:39:45,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497. as already flushing 2024-11-17T15:39:45,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497. 2024-11-17T15:39:45,059 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => a682ec5138f7084a1f2f9cd3afa94497, NAME => 'TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:39:45,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => a682ec5138f7084a1f2f9cd3afa94497, NAME => 'TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:39:45,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a682ec5138f7084a1f2f9cd3afa94497, NAME => 'TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a682ec5138f7084a1f2f9cd3afa94497, NAME => 'TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:39:45,212 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-17T15:39:45,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497. 2024-11-17T15:39:45,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497. as already flushing 2024-11-17T15:39:45,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497. 2024-11-17T15:39:45,212 ERROR [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => a682ec5138f7084a1f2f9cd3afa94497, NAME => 'TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:39:45,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => a682ec5138f7084a1f2f9cd3afa94497, NAME => 'TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:39:45,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a682ec5138f7084a1f2f9cd3afa94497, NAME => 'TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a682ec5138f7084a1f2f9cd3afa94497, NAME => 'TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T15:39:45,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-17T15:39:45,240 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=321 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/C/c7954d30fec540a885f670a7821e9690 2024-11-17T15:39:45,244 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/A/65e0522a067240c1bf0553ec1bf65235 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/A/65e0522a067240c1bf0553ec1bf65235 2024-11-17T15:39:45,247 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/A/65e0522a067240c1bf0553ec1bf65235, entries=250, sequenceid=321, filesize=47.5 K 2024-11-17T15:39:45,247 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/B/df4115e30f234f84a9eb1282e785e0fb as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/B/df4115e30f234f84a9eb1282e785e0fb 2024-11-17T15:39:45,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,250 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/B/df4115e30f234f84a9eb1282e785e0fb, entries=150, sequenceid=321, filesize=12.0 K 2024-11-17T15:39:45,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,252 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/C/c7954d30fec540a885f670a7821e9690 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/C/c7954d30fec540a885f670a7821e9690 2024-11-17T15:39:45,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,256 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/C/c7954d30fec540a885f670a7821e9690, entries=150, sequenceid=321, filesize=12.0 K 2024-11-17T15:39:45,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,256 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for a682ec5138f7084a1f2f9cd3afa94497 in 1378ms, sequenceid=321, compaction requested=true 2024-11-17T15:39:45,256 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a682ec5138f7084a1f2f9cd3afa94497: 2024-11-17T15:39:45,256 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a682ec5138f7084a1f2f9cd3afa94497:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T15:39:45,256 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T15:39:45,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,256 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a682ec5138f7084a1f2f9cd3afa94497:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T15:39:45,256 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T15:39:45,256 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T15:39:45,257 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a682ec5138f7084a1f2f9cd3afa94497:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T15:39:45,257 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T15:39:45,257 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T15:39:45,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,257 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 172653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T15:39:45,257 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T15:39:45,257 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1541): a682ec5138f7084a1f2f9cd3afa94497/A is initiating minor compaction (all files) 2024-11-17T15:39:45,257 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1541): a682ec5138f7084a1f2f9cd3afa94497/B is initiating minor compaction (all files) 2024-11-17T15:39:45,257 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a682ec5138f7084a1f2f9cd3afa94497/A in TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497. 2024-11-17T15:39:45,257 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a682ec5138f7084a1f2f9cd3afa94497/B in TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497. 2024-11-17T15:39:45,258 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/A/ec79ac6bdc6643f3a56d28cbceee8373, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/A/67a8fcefd62347529545281f4f125f5b, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/A/65e0522a067240c1bf0553ec1bf65235] into tmpdir=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp, totalSize=168.6 K 2024-11-17T15:39:45,258 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497. 2024-11-17T15:39:45,258 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/B/08d112961f57431b8285e2d257fa1ace, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/B/acf9528f40ab418ca11bc1f10ced4af8, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/B/df4115e30f234f84a9eb1282e785e0fb] into tmpdir=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp, totalSize=36.6 K 2024-11-17T15:39:45,258 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497. files: [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/A/ec79ac6bdc6643f3a56d28cbceee8373, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/A/67a8fcefd62347529545281f4f125f5b, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/A/65e0522a067240c1bf0553ec1bf65235] 2024-11-17T15:39:45,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,258 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting 08d112961f57431b8285e2d257fa1ace, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1731857982235 2024-11-17T15:39:45,258 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting ec79ac6bdc6643f3a56d28cbceee8373, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1731857982235 2024-11-17T15:39:45,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,258 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting acf9528f40ab418ca11bc1f10ced4af8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1731857983716 2024-11-17T15:39:45,259 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 67a8fcefd62347529545281f4f125f5b, keycount=500, bloomtype=ROW, size=90.0 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1731857983385 2024-11-17T15:39:45,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,259 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting df4115e30f234f84a9eb1282e785e0fb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=321, earliestPutTs=1731857983757 2024-11-17T15:39:45,259 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 65e0522a067240c1bf0553ec1bf65235, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=321, earliestPutTs=1731857983757 2024-11-17T15:39:45,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,272 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a682ec5138f7084a1f2f9cd3afa94497#B#compaction#509 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T15:39:45,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,273 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/B/abfb95d6e1c94568aa9ad8260d173035 is 50, key is test_row_0/B:col10/1731857983757/Put/seqid=0 2024-11-17T15:39:45,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,274 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a682ec5138f7084a1f2f9cd3afa94497] 2024-11-17T15:39:45,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,276 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411170343e2eea343477c89c044a6b521d79a_a682ec5138f7084a1f2f9cd3afa94497 store=[table=TestAcidGuarantees family=A region=a682ec5138f7084a1f2f9cd3afa94497] 2024-11-17T15:39:45,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,291 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411170343e2eea343477c89c044a6b521d79a_a682ec5138f7084a1f2f9cd3afa94497, store=[table=TestAcidGuarantees family=A region=a682ec5138f7084a1f2f9cd3afa94497] 2024-11-17T15:39:45,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,291 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411170343e2eea343477c89c044a6b521d79a_a682ec5138f7084a1f2f9cd3afa94497 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a682ec5138f7084a1f2f9cd3afa94497] 2024-11-17T15:39:45,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742434_1610 (size=12983) 2024-11-17T15:39:45,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742435_1611 (size=4469) 2024-11-17T15:39:45,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,330 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a682ec5138f7084a1f2f9cd3afa94497#A#compaction#510 average throughput is 0.44 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T15:39:45,331 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/A/7d00fcbaffeb43c7a1d6887f9967e320 is 175, key is test_row_0/A:col10/1731857983757/Put/seqid=0 2024-11-17T15:39:45,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742436_1612 (size=31937) 2024-11-17T15:39:45,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,366 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-17T15:39:45,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497. 2024-11-17T15:39:45,366 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2902): Flushing a682ec5138f7084a1f2f9cd3afa94497 3/3 column families, dataSize=40.25 KB heapSize=106.22 KB 2024-11-17T15:39:45,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411171b14f80943c44cad88939a15a871990c_a682ec5138f7084a1f2f9cd3afa94497 is 50, key is test_row_0/A:col10/1731857983891/Put/seqid=0 2024-11-17T15:39:45,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742437_1613 (size=9914) 2024-11-17T15:39:45,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-17T15:39:45,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,698 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/B/abfb95d6e1c94568aa9ad8260d173035 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/B/abfb95d6e1c94568aa9ad8260d173035 2024-11-17T15:39:45,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,704 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a682ec5138f7084a1f2f9cd3afa94497/B of a682ec5138f7084a1f2f9cd3afa94497 into abfb95d6e1c94568aa9ad8260d173035(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T15:39:45,704 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a682ec5138f7084a1f2f9cd3afa94497: 2024-11-17T15:39:45,704 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., storeName=a682ec5138f7084a1f2f9cd3afa94497/B, priority=13, startTime=1731857985256; duration=0sec 2024-11-17T15:39:45,704 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T15:39:45,704 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a682ec5138f7084a1f2f9cd3afa94497:B 2024-11-17T15:39:45,704 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T15:39:45,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,705 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T15:39:45,705 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1541): a682ec5138f7084a1f2f9cd3afa94497/C is initiating minor compaction (all files) 2024-11-17T15:39:45,705 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a682ec5138f7084a1f2f9cd3afa94497/C in TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497. 2024-11-17T15:39:45,705 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/C/135dff1cf42241348acb4c354a76866c, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/C/34c973e46037411f8d07d43dbb49b98d, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/C/c7954d30fec540a885f670a7821e9690] into tmpdir=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp, totalSize=36.6 K 2024-11-17T15:39:45,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,706 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting 135dff1cf42241348acb4c354a76866c, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1731857982235 2024-11-17T15:39:45,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,706 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting 34c973e46037411f8d07d43dbb49b98d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1731857983716 2024-11-17T15:39:45,706 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] compactions.Compactor(225): Compacting c7954d30fec540a885f670a7821e9690, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=321, earliestPutTs=1731857983757 2024-11-17T15:39:45,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,714 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a682ec5138f7084a1f2f9cd3afa94497#C#compaction#512 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T15:39:45,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,714 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/C/9d5bc5340abf49c0a5e32f676c30e3d0 is 50, key is test_row_0/C:col10/1731857983757/Put/seqid=0 2024-11-17T15:39:45,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742438_1614 (size=12983) 2024-11-17T15:39:45,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,725 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/C/9d5bc5340abf49c0a5e32f676c30e3d0 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/C/9d5bc5340abf49c0a5e32f676c30e3d0 2024-11-17T15:39:45,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,731 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a682ec5138f7084a1f2f9cd3afa94497/C of a682ec5138f7084a1f2f9cd3afa94497 into 9d5bc5340abf49c0a5e32f676c30e3d0(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T15:39:45,731 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a682ec5138f7084a1f2f9cd3afa94497: 2024-11-17T15:39:45,731 INFO [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., storeName=a682ec5138f7084a1f2f9cd3afa94497/C, priority=13, startTime=1731857985257; duration=0sec 2024-11-17T15:39:45,731 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T15:39:45,731 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a682ec5138f7084a1f2f9cd3afa94497:C 2024-11-17T15:39:45,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,747 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/A/7d00fcbaffeb43c7a1d6887f9967e320 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/A/7d00fcbaffeb43c7a1d6887f9967e320 2024-11-17T15:39:45,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,751 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a682ec5138f7084a1f2f9cd3afa94497/A of a682ec5138f7084a1f2f9cd3afa94497 into 7d00fcbaffeb43c7a1d6887f9967e320(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T15:39:45,751 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a682ec5138f7084a1f2f9cd3afa94497: 2024-11-17T15:39:45,751 INFO [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., storeName=a682ec5138f7084a1f2f9cd3afa94497/A, priority=13, startTime=1731857985256; duration=0sec 2024-11-17T15:39:45,752 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T15:39:45,752 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a682ec5138f7084a1f2f9cd3afa94497:A 2024-11-17T15:39:45,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,782 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411171b14f80943c44cad88939a15a871990c_a682ec5138f7084a1f2f9cd3afa94497 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411171b14f80943c44cad88939a15a871990c_a682ec5138f7084a1f2f9cd3afa94497 2024-11-17T15:39:45,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/A/4d930bf6cfc741bfb1502ee70410e461, store: [table=TestAcidGuarantees family=A region=a682ec5138f7084a1f2f9cd3afa94497] 2024-11-17T15:39:45,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/A/4d930bf6cfc741bfb1502ee70410e461 is 175, key is test_row_0/A:col10/1731857983891/Put/seqid=0 2024-11-17T15:39:45,791 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742439_1615 (size=22561) 2024-11-17T15:39:45,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:45,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:46,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:46,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:46,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:46,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:46,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:46,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:46,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:46,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:46,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:46,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:46,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:46,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:46,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:46,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:46,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:46,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:46,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:46,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:46,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:46,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:46,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:46,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:46,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:46,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:46,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:46,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:46,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:46,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:46,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:46,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:46,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:46,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:46,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:46,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:46,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:46,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:46,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:46,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:46,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:46,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:46,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:46,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:46,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:46,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:46,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:46,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:46,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:46,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:46,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:46,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:46,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:46,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:46,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:46,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:46,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:46,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:46,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:46,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:46,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:46,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:46,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:46,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:46,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:46,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:46,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:46,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:46,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:46,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:46,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:46,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-17T15:39:46,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:46,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:46,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:46,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(8855): Flush requested on a682ec5138f7084a1f2f9cd3afa94497 2024-11-17T15:39:46,046 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497. as already flushing 2024-11-17T15:39:46,082 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:46,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57374 deadline: 1731858046076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:46,083 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:46,083 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:46,083 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:46,084 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:46,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57310 deadline: 1731858046079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:46,084 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:46,084 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:46,084 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:46,084 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:46,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57332 deadline: 1731858046080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:46,085 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:46,085 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:46,085 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:46,086 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:46,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57380 deadline: 1731858046083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:46,086 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:46,086 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:46,086 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:46,192 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=330, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/A/4d930bf6cfc741bfb1502ee70410e461 2024-11-17T15:39:46,197 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:46,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57374 deadline: 1731858046194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:46,198 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:46,198 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:46,198 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:46,198 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:46,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57310 deadline: 1731858046194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:46,198 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:46,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57380 deadline: 1731858046194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:46,198 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:46,198 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:46,198 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:46,198 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:46,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57332 deadline: 1731858046194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:46,198 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:46,198 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:46,198 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:46,198 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:46,198 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:46,198 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:46,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/B/e4684a5ec56a4ba89b8c98266f3b0446 is 50, key is test_row_0/B:col10/1731857983891/Put/seqid=0 2024-11-17T15:39:46,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742440_1616 (size=9857) 2024-11-17T15:39:46,405 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:46,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57374 deadline: 1731858046403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:46,405 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:46,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57380 deadline: 1731858046403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:46,406 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:46,406 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:46,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57310 deadline: 1731858046403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:46,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57332 deadline: 1731858046403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:46,406 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:46,406 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:46,406 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:46,406 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:46,406 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:46,406 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:46,406 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:46,406 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:46,406 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:46,406 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:46,406 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:46,406 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:46,434 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:46,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57288 deadline: 1731858046433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:46,435 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 7a780d55532c,32955,1731857800648 for 'test_row_0' in a682ec5138f7084a1f2f9cd3afa94497 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4202 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T15:39:46,436 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:46,436 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:46,436 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:46,603 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=330 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/B/e4684a5ec56a4ba89b8c98266f3b0446 2024-11-17T15:39:46,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/C/4177325527e14e93a99e63d09d2fca17 is 50, key is test_row_0/C:col10/1731857983891/Put/seqid=0 2024-11-17T15:39:46,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742441_1617 (size=9857) 2024-11-17T15:39:46,716 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:46,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57374 deadline: 1731858046713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:46,716 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:46,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57310 deadline: 1731858046713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:46,717 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:46,717 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:46,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57380 deadline: 1731858046713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:46,717 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:46,717 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:46,717 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:46,717 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:46,717 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:46,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57332 deadline: 1731858046713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:46,717 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:46,717 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:46,717 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:46,717 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:46,717 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:46,717 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:46,717 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:47,018 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=330 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/C/4177325527e14e93a99e63d09d2fca17 2024-11-17T15:39:47,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/A/4d930bf6cfc741bfb1502ee70410e461 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/A/4d930bf6cfc741bfb1502ee70410e461 2024-11-17T15:39:47,025 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/A/4d930bf6cfc741bfb1502ee70410e461, entries=100, sequenceid=330, filesize=22.0 K 2024-11-17T15:39:47,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/B/e4684a5ec56a4ba89b8c98266f3b0446 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/B/e4684a5ec56a4ba89b8c98266f3b0446 2024-11-17T15:39:47,029 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/B/e4684a5ec56a4ba89b8c98266f3b0446, entries=100, sequenceid=330, filesize=9.6 K 2024-11-17T15:39:47,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/C/4177325527e14e93a99e63d09d2fca17 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/C/4177325527e14e93a99e63d09d2fca17 2024-11-17T15:39:47,033 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/C/4177325527e14e93a99e63d09d2fca17, entries=100, sequenceid=330, filesize=9.6 K 2024-11-17T15:39:47,034 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.17 KB/108720, currentSize=161.02 KB/164880 for a682ec5138f7084a1f2f9cd3afa94497 in 1668ms, sequenceid=330, compaction requested=false 2024-11-17T15:39:47,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2603): Flush status journal for a682ec5138f7084a1f2f9cd3afa94497: 2024-11-17T15:39:47,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497. 2024-11-17T15:39:47,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=165 2024-11-17T15:39:47,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.HMaster(4169): Remote procedure done, pid=165 2024-11-17T15:39:47,041 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=165, resume processing ppid=164 2024-11-17T15:39:47,041 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=165, ppid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1280 sec 2024-11-17T15:39:47,043 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees in 2.1380 sec 2024-11-17T15:39:47,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-17T15:39:47,053 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T15:39:47,054 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T15:39:47,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] procedure2.ProcedureExecutor(1139): Stored pid=166, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees 2024-11-17T15:39:47,055 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=166, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T15:39:47,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-17T15:39:47,056 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=166, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T15:39:47,056 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=167, ppid=166, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T15:39:47,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-17T15:39:47,208 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-17T15:39:47,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497. 2024-11-17T15:39:47,208 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2902): Flushing a682ec5138f7084a1f2f9cd3afa94497 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-17T15:39:47,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411172d9af8688bf44d039c379f7108218a0f_a682ec5138f7084a1f2f9cd3afa94497 is 50, key is test_row_0/A:col10/1731857986072/Put/seqid=0 2024-11-17T15:39:47,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742442_1618 (size=12454) 2024-11-17T15:39:47,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(8855): Flush requested on a682ec5138f7084a1f2f9cd3afa94497 2024-11-17T15:39:47,225 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497. as already flushing 2024-11-17T15:39:47,232 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:47,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57332 deadline: 1731858047228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:47,232 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:47,232 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:47,232 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:47,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57380 deadline: 1731858047228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:47,232 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:47,233 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:47,233 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:47,233 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:47,236 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:47,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57374 deadline: 1731858047231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:47,236 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:47,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57310 deadline: 1731858047232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:47,236 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:47,236 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:47,236 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:47,236 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:47,236 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:47,236 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:47,346 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:47,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57332 deadline: 1731858047343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:47,346 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:47,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57380 deadline: 1731858047343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:47,346 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:47,346 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:47,347 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:47,347 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:47,347 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:47,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57374 deadline: 1731858047344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:47,347 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:47,347 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:47,347 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:47,347 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:47,347 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:47,347 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:47,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57310 deadline: 1731858047344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:47,347 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:47,347 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:47,347 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:47,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-17T15:39:47,555 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:47,555 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:47,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57332 deadline: 1731858047553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:47,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57380 deadline: 1731858047553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:47,556 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:47,556 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:47,556 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:47,556 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:47,556 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:47,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57374 deadline: 1731858047553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:47,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57310 deadline: 1731858047553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:47,556 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:47,556 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:47,556 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:47,556 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:47,556 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:47,556 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:47,556 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:47,556 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:47,556 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:47,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:47,623 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411172d9af8688bf44d039c379f7108218a0f_a682ec5138f7084a1f2f9cd3afa94497 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411172d9af8688bf44d039c379f7108218a0f_a682ec5138f7084a1f2f9cd3afa94497 2024-11-17T15:39:47,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/A/347fe25c66c343b595fadd951715270a, store: [table=TestAcidGuarantees family=A region=a682ec5138f7084a1f2f9cd3afa94497] 2024-11-17T15:39:47,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/A/347fe25c66c343b595fadd951715270a is 175, key is test_row_0/A:col10/1731857986072/Put/seqid=0 2024-11-17T15:39:47,628 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742443_1619 (size=31255) 2024-11-17T15:39:47,628 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=360, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/A/347fe25c66c343b595fadd951715270a 2024-11-17T15:39:47,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/B/5bd1b00ecbb14a21bf905726c84bf16b is 50, key is test_row_0/B:col10/1731857986072/Put/seqid=0 2024-11-17T15:39:47,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742444_1620 (size=12301) 2024-11-17T15:39:47,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-17T15:39:47,865 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:47,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57332 deadline: 1731858047863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:47,865 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:47,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57310 deadline: 1731858047863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:47,866 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:47,866 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:47,866 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:47,866 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:47,866 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:47,866 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:47,868 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:47,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57374 deadline: 1731858047864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:47,868 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:47,869 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:47,869 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:47,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57380 deadline: 1731858047864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:47,869 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:47,869 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:47,869 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:47,869 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:48,046 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=360 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/B/5bd1b00ecbb14a21bf905726c84bf16b 2024-11-17T15:39:48,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/C/7eb3d058f5ce48ac9ce823525a16c2ed is 50, key is test_row_0/C:col10/1731857986072/Put/seqid=0 2024-11-17T15:39:48,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742445_1621 (size=12301) 2024-11-17T15:39:48,057 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=360 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/C/7eb3d058f5ce48ac9ce823525a16c2ed 2024-11-17T15:39:48,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/A/347fe25c66c343b595fadd951715270a as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/A/347fe25c66c343b595fadd951715270a 2024-11-17T15:39:48,071 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/A/347fe25c66c343b595fadd951715270a, entries=150, sequenceid=360, filesize=30.5 K 2024-11-17T15:39:48,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/B/5bd1b00ecbb14a21bf905726c84bf16b as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/B/5bd1b00ecbb14a21bf905726c84bf16b 2024-11-17T15:39:48,075 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/B/5bd1b00ecbb14a21bf905726c84bf16b, entries=150, sequenceid=360, filesize=12.0 K 2024-11-17T15:39:48,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/C/7eb3d058f5ce48ac9ce823525a16c2ed as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/C/7eb3d058f5ce48ac9ce823525a16c2ed 2024-11-17T15:39:48,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,082 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/C/7eb3d058f5ce48ac9ce823525a16c2ed, entries=150, sequenceid=360, filesize=12.0 K 2024-11-17T15:39:48,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,083 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for a682ec5138f7084a1f2f9cd3afa94497 in 875ms, sequenceid=360, compaction requested=true 2024-11-17T15:39:48,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2603): Flush status journal for a682ec5138f7084a1f2f9cd3afa94497: 2024-11-17T15:39:48,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497. 2024-11-17T15:39:48,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=167 2024-11-17T15:39:48,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.HMaster(4169): Remote procedure done, pid=167 2024-11-17T15:39:48,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,086 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=167, resume processing ppid=166 2024-11-17T15:39:48,086 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=167, ppid=166, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0280 sec 2024-11-17T15:39:48,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,088 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=166, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees in 1.0330 sec 2024-11-17T15:39:48,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-17T15:39:48,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,193 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T15:39:48,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,194 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T15:39:48,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] procedure2.ProcedureExecutor(1139): Stored pid=168, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees 2024-11-17T15:39:48,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-17T15:39:48,196 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=168, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T15:39:48,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,196 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=168, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T15:39:48,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,197 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=169, ppid=168, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T15:39:48,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-17T15:39:48,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,349 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-17T15:39:48,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497. 2024-11-17T15:39:48,349 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2902): Flushing a682ec5138f7084a1f2f9cd3afa94497 3/3 column families, dataSize=40.25 KB heapSize=106.22 KB 2024-11-17T15:39:48,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111767a86801faef4ad7a212089a2f17ce7d_a682ec5138f7084a1f2f9cd3afa94497 is 50, key is test_row_0/A:col10/1731857987230/Put/seqid=0 2024-11-17T15:39:48,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742446_1622 (size=9914) 2024-11-17T15:39:48,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,364 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111767a86801faef4ad7a212089a2f17ce7d_a682ec5138f7084a1f2f9cd3afa94497 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111767a86801faef4ad7a212089a2f17ce7d_a682ec5138f7084a1f2f9cd3afa94497 2024-11-17T15:39:48,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/A/fedf2291c3734c7782107da4a45b4489, store: [table=TestAcidGuarantees family=A region=a682ec5138f7084a1f2f9cd3afa94497] 2024-11-17T15:39:48,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/A/fedf2291c3734c7782107da4a45b4489 is 175, key is test_row_0/A:col10/1731857987230/Put/seqid=0 2024-11-17T15:39:48,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742447_1623 (size=22561) 2024-11-17T15:39:48,370 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=369, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/A/fedf2291c3734c7782107da4a45b4489 2024-11-17T15:39:48,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(8855): Flush requested on a682ec5138f7084a1f2f9cd3afa94497 2024-11-17T15:39:48,382 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497. as already flushing 2024-11-17T15:39:48,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/B/a43eec3f55ec44c99e823369123d24dd is 50, key is test_row_0/B:col10/1731857987230/Put/seqid=0 2024-11-17T15:39:48,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742448_1624 (size=9857) 2024-11-17T15:39:48,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:48,439 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:48,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57332 deadline: 1731858048434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:48,439 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:48,439 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:48,440 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:48,440 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:48,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57310 deadline: 1731858048434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:48,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57374 deadline: 1731858048435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:48,440 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:48,440 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:48,440 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:48,440 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:48,440 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:48,440 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:48,440 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:48,443 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:48,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57380 deadline: 1731858048439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:48,443 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:48,443 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:48,443 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:48,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-17T15:39:48,544 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:48,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57332 deadline: 1731858048543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:48,545 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:48,545 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:48,545 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:48,547 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:48,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57374 deadline: 1731858048543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:48,547 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:48,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57310 deadline: 1731858048543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:48,548 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:48,548 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:48,548 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:48,548 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:48,548 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:48,548 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:48,554 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:48,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57380 deadline: 1731858048553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:48,554 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:48,554 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:48,554 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:48,755 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:48,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57310 deadline: 1731858048753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:48,756 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:48,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57332 deadline: 1731858048753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:48,756 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:48,756 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:48,756 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:48,756 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:48,756 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:48,756 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:48,756 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:48,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57374 deadline: 1731858048753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:48,756 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:48,756 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:48,756 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:48,765 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:48,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57380 deadline: 1731858048763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:48,766 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:48,766 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:48,766 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:48,798 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=369 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/B/a43eec3f55ec44c99e823369123d24dd 2024-11-17T15:39:48,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/C/d2ac9f1b37ab4699bf8a9256f00e9c16 is 50, key is test_row_0/C:col10/1731857987230/Put/seqid=0 2024-11-17T15:39:48,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742449_1625 (size=9857) 2024-11-17T15:39:48,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-17T15:39:48,831 INFO [Thread-2389 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2389. 2024-11-17T15:39:48,831 DEBUG [Thread-2389 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T15:39:48,832 DEBUG [Thread-2389 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:39:48,832 DEBUG [Thread-2389 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:39:48,832 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T15:39:48,835 INFO [Thread-2387 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2387. 2024-11-17T15:39:48,835 DEBUG [Thread-2387 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T15:39:48,835 DEBUG [Thread-2387 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:39:48,835 DEBUG [Thread-2387 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:39:48,835 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T15:39:48,835 INFO [Thread-2385 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2385. 2024-11-17T15:39:48,835 DEBUG [Thread-2385 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T15:39:48,835 DEBUG [Thread-2385 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:39:48,836 DEBUG [Thread-2385 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:39:48,836 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T15:39:48,836 INFO [Thread-2381 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2381. 2024-11-17T15:39:48,836 DEBUG [Thread-2381 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T15:39:48,836 DEBUG [Thread-2381 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:39:48,836 DEBUG [Thread-2381 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:39:48,836 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T15:39:48,837 INFO [Thread-2383 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2383. 2024-11-17T15:39:48,837 DEBUG [Thread-2383 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T15:39:48,837 DEBUG [Thread-2383 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:39:48,837 DEBUG [Thread-2383 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:39:48,837 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T15:39:49,063 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:49,063 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:49,063 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:49,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57310 deadline: 1731858049063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:49,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57332 deadline: 1731858049063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:49,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57374 deadline: 1731858049063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:49,064 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:49,064 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:49,064 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:49,064 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:49,064 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:49,064 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:49,064 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:49,064 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:49,064 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:49,073 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T15:39:49,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57380 deadline: 1731858049073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 2024-11-17T15:39:49,074 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 , the old value is region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:49,074 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T15:39:49,074 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., hostname=7a780d55532c,32955,1731857800648, seqNum=5 because the exception is null or not the one we care about 2024-11-17T15:39:49,212 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=369 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/C/d2ac9f1b37ab4699bf8a9256f00e9c16 2024-11-17T15:39:49,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/A/fedf2291c3734c7782107da4a45b4489 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/A/fedf2291c3734c7782107da4a45b4489 2024-11-17T15:39:49,217 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/A/fedf2291c3734c7782107da4a45b4489, entries=100, sequenceid=369, filesize=22.0 K 2024-11-17T15:39:49,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/B/a43eec3f55ec44c99e823369123d24dd as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/B/a43eec3f55ec44c99e823369123d24dd 2024-11-17T15:39:49,220 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/B/a43eec3f55ec44c99e823369123d24dd, entries=100, sequenceid=369, filesize=9.6 K 2024-11-17T15:39:49,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/C/d2ac9f1b37ab4699bf8a9256f00e9c16 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/C/d2ac9f1b37ab4699bf8a9256f00e9c16 2024-11-17T15:39:49,223 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/C/d2ac9f1b37ab4699bf8a9256f00e9c16, entries=100, sequenceid=369, filesize=9.6 K 2024-11-17T15:39:49,224 INFO [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.17 KB/108720, currentSize=161.02 KB/164880 for a682ec5138f7084a1f2f9cd3afa94497 in 875ms, sequenceid=369, compaction requested=true 2024-11-17T15:39:49,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2603): Flush status journal for a682ec5138f7084a1f2f9cd3afa94497: 2024-11-17T15:39:49,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497. 2024-11-17T15:39:49,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7a780d55532c:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=169 2024-11-17T15:39:49,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.HMaster(4169): Remote procedure done, pid=169 2024-11-17T15:39:49,226 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=169, resume processing ppid=168 2024-11-17T15:39:49,226 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=169, ppid=168, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0280 sec 2024-11-17T15:39:49,228 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=168, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees in 1.0330 sec 2024-11-17T15:39:49,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-17T15:39:49,333 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T15:39:49,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32955 {}] regionserver.HRegion(8855): Flush requested on a682ec5138f7084a1f2f9cd3afa94497 2024-11-17T15:39:49,574 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T15:39:49,575 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a682ec5138f7084a1f2f9cd3afa94497 3/3 column families, dataSize=181.14 KB heapSize=475.36 KB 2024-11-17T15:39:49,575 INFO [Thread-2374 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2374. 2024-11-17T15:39:49,575 INFO [Thread-2376 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2376. 2024-11-17T15:39:49,575 INFO [Thread-2370 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2370. 2024-11-17T15:39:49,575 DEBUG [Thread-2374 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T15:39:49,575 DEBUG [Thread-2376 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T15:39:49,575 DEBUG [Thread-2370 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T15:39:49,575 DEBUG [Thread-2374 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:39:49,575 DEBUG [Thread-2376 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:39:49,575 DEBUG [Thread-2370 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:39:49,575 DEBUG [Thread-2376 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:39:49,575 DEBUG [Thread-2374 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:39:49,575 DEBUG [Thread-2370 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:39:49,575 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T15:39:49,575 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T15:39:49,575 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T15:39:49,580 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411176fce87b261414263be509354836f9610_a682ec5138f7084a1f2f9cd3afa94497 is 50, key is test_row_0/A:col10/1731857989573/Put/seqid=0 2024-11-17T15:39:49,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742450_1626 (size=12454) 2024-11-17T15:39:49,584 INFO [Thread-2378 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2378. 2024-11-17T15:39:49,584 DEBUG [Thread-2378 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T15:39:49,584 DEBUG [Thread-2378 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:39:49,584 DEBUG [Thread-2378 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:39:49,584 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T15:39:49,984 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:49,986 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411176fce87b261414263be509354836f9610_a682ec5138f7084a1f2f9cd3afa94497 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411176fce87b261414263be509354836f9610_a682ec5138f7084a1f2f9cd3afa94497 2024-11-17T15:39:49,987 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/A/6bef57857aaf41a9852641b7ff78ba0c, store: [table=TestAcidGuarantees family=A region=a682ec5138f7084a1f2f9cd3afa94497] 2024-11-17T15:39:49,987 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/A/6bef57857aaf41a9852641b7ff78ba0c is 175, key is test_row_0/A:col10/1731857989573/Put/seqid=0 2024-11-17T15:39:49,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742451_1627 (size=31255) 2024-11-17T15:39:50,391 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=399, memsize=60.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/A/6bef57857aaf41a9852641b7ff78ba0c 2024-11-17T15:39:50,396 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/B/5487658398544cc9b11a1ff9cb55a05f is 50, key is test_row_0/B:col10/1731857989573/Put/seqid=0 2024-11-17T15:39:50,399 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742452_1628 (size=12301) 2024-11-17T15:39:50,465 INFO [Thread-2372 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2372. 2024-11-17T15:39:50,465 DEBUG [Thread-2372 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T15:39:50,465 DEBUG [Thread-2372 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:39:50,465 DEBUG [Thread-2372 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:39:50,465 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T15:39:50,465 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-17T15:39:50,465 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 61 2024-11-17T15:39:50,465 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 46 2024-11-17T15:39:50,465 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 68 2024-11-17T15:39:50,465 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 72 2024-11-17T15:39:50,465 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 75 2024-11-17T15:39:50,465 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-17T15:39:50,465 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3085 2024-11-17T15:39:50,465 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2961 2024-11-17T15:39:50,466 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2963 2024-11-17T15:39:50,466 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3087 2024-11-17T15:39:50,466 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2979 2024-11-17T15:39:50,466 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-17T15:39:50,466 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-17T15:39:50,466 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-17T15:39:50,466 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:39:50,466 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:39:50,466 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T15:39:50,467 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-17T15:39:50,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] procedure2.ProcedureExecutor(1139): Stored pid=170, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-17T15:39:50,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-17T15:39:50,469 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731857990469"}]},"ts":"1731857990469"} 2024-11-17T15:39:50,470 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-17T15:39:50,470 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-17T15:39:50,471 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=171, ppid=170, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-17T15:39:50,472 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=172, ppid=171, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a682ec5138f7084a1f2f9cd3afa94497, UNASSIGN}] 2024-11-17T15:39:50,472 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=172, ppid=171, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a682ec5138f7084a1f2f9cd3afa94497, UNASSIGN 2024-11-17T15:39:50,473 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=172 updating hbase:meta row=a682ec5138f7084a1f2f9cd3afa94497, regionState=CLOSING, regionLocation=7a780d55532c,32955,1731857800648 2024-11-17T15:39:50,474 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=172, ppid=171, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a682ec5138f7084a1f2f9cd3afa94497, UNASSIGN because future has completed 2024-11-17T15:39:50,474 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-17T15:39:50,474 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=173, ppid=172, state=RUNNABLE, hasLock=false; CloseRegionProcedure a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648}] 2024-11-17T15:39:50,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-17T15:39:50,626 INFO [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] handler.UnassignRegionHandler(122): Close a682ec5138f7084a1f2f9cd3afa94497 2024-11-17T15:39:50,627 DEBUG [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-17T15:39:50,627 DEBUG [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegion(1722): Closing a682ec5138f7084a1f2f9cd3afa94497, disabling compactions & flushes 2024-11-17T15:39:50,627 DEBUG [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497. 2024-11-17T15:39:50,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-17T15:39:50,800 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=399 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/B/5487658398544cc9b11a1ff9cb55a05f 2024-11-17T15:39:50,806 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/C/16d78d7208684a24bf57057f37d67b32 is 50, key is test_row_0/C:col10/1731857989573/Put/seqid=0 2024-11-17T15:39:50,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742453_1629 (size=12301) 2024-11-17T15:39:51,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-17T15:39:51,210 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=399 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/C/16d78d7208684a24bf57057f37d67b32 2024-11-17T15:39:51,213 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/A/6bef57857aaf41a9852641b7ff78ba0c as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/A/6bef57857aaf41a9852641b7ff78ba0c 2024-11-17T15:39:51,215 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/A/6bef57857aaf41a9852641b7ff78ba0c, entries=150, sequenceid=399, filesize=30.5 K 2024-11-17T15:39:51,215 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/B/5487658398544cc9b11a1ff9cb55a05f as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/B/5487658398544cc9b11a1ff9cb55a05f 2024-11-17T15:39:51,218 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/B/5487658398544cc9b11a1ff9cb55a05f, entries=150, sequenceid=399, filesize=12.0 K 2024-11-17T15:39:51,218 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/C/16d78d7208684a24bf57057f37d67b32 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/C/16d78d7208684a24bf57057f37d67b32 2024-11-17T15:39:51,221 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/C/16d78d7208684a24bf57057f37d67b32, entries=150, sequenceid=399, filesize=12.0 K 2024-11-17T15:39:51,222 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~181.14 KB/185490, heapSize ~475.31 KB/486720, currentSize=13.42 KB/13740 for a682ec5138f7084a1f2f9cd3afa94497 in 1648ms, sequenceid=399, compaction requested=true 2024-11-17T15:39:51,222 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a682ec5138f7084a1f2f9cd3afa94497: 2024-11-17T15:39:51,222 INFO [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497. 2024-11-17T15:39:51,222 DEBUG [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497. 2024-11-17T15:39:51,222 DEBUG [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497. after waiting 0 ms 2024-11-17T15:39:51,222 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a682ec5138f7084a1f2f9cd3afa94497:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T15:39:51,222 DEBUG [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497. 2024-11-17T15:39:51,222 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T15:39:51,222 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497. because compaction request was cancelled 2024-11-17T15:39:51,222 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a682ec5138f7084a1f2f9cd3afa94497:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T15:39:51,222 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a682ec5138f7084a1f2f9cd3afa94497:A 2024-11-17T15:39:51,222 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T15:39:51,222 INFO [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegion(2902): Flushing a682ec5138f7084a1f2f9cd3afa94497 3/3 column families, dataSize=13.42 KB heapSize=35.91 KB 2024-11-17T15:39:51,222 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497. because compaction request was cancelled 2024-11-17T15:39:51,222 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a682ec5138f7084a1f2f9cd3afa94497:C, priority=-2147483648, current under compaction store size is 2 2024-11-17T15:39:51,222 DEBUG [RS:0;7a780d55532c:32955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a682ec5138f7084a1f2f9cd3afa94497:B 2024-11-17T15:39:51,222 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497. because compaction request was cancelled 2024-11-17T15:39:51,222 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T15:39:51,222 DEBUG [RS:0;7a780d55532c:32955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a682ec5138f7084a1f2f9cd3afa94497:C 2024-11-17T15:39:51,226 DEBUG [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411170e6f8c330161497fb1295bcee41aa385_a682ec5138f7084a1f2f9cd3afa94497 is 50, key is test_row_0/A:col10/1731857990464/Put/seqid=0 2024-11-17T15:39:51,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742454_1630 (size=7374) 2024-11-17T15:39:51,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-17T15:39:51,630 DEBUG [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T15:39:51,632 INFO [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411170e6f8c330161497fb1295bcee41aa385_a682ec5138f7084a1f2f9cd3afa94497 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411170e6f8c330161497fb1295bcee41aa385_a682ec5138f7084a1f2f9cd3afa94497 2024-11-17T15:39:51,633 DEBUG [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/A/d9d1ef44d1684eadb1b2105b5044e4ce, store: [table=TestAcidGuarantees family=A region=a682ec5138f7084a1f2f9cd3afa94497] 2024-11-17T15:39:51,634 DEBUG [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/A/d9d1ef44d1684eadb1b2105b5044e4ce is 175, key is test_row_0/A:col10/1731857990464/Put/seqid=0 2024-11-17T15:39:51,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742455_1631 (size=13865) 2024-11-17T15:39:52,037 INFO [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=404, memsize=4.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/A/d9d1ef44d1684eadb1b2105b5044e4ce 2024-11-17T15:39:52,043 DEBUG [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/B/e40e83761dad438dac03051467d7c475 is 50, key is test_row_0/B:col10/1731857990464/Put/seqid=0 2024-11-17T15:39:52,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742456_1632 (size=7415) 2024-11-17T15:39:52,446 INFO [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=404 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/B/e40e83761dad438dac03051467d7c475 2024-11-17T15:39:52,451 DEBUG [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/C/2eee554e2bb1407395c9fa199f0a54a8 is 50, key is test_row_0/C:col10/1731857990464/Put/seqid=0 2024-11-17T15:39:52,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742457_1633 (size=7415) 2024-11-17T15:39:52,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-17T15:39:52,855 INFO [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=404 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/C/2eee554e2bb1407395c9fa199f0a54a8 2024-11-17T15:39:52,858 DEBUG [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/A/d9d1ef44d1684eadb1b2105b5044e4ce as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/A/d9d1ef44d1684eadb1b2105b5044e4ce 2024-11-17T15:39:52,860 INFO [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/A/d9d1ef44d1684eadb1b2105b5044e4ce, entries=50, sequenceid=404, filesize=13.5 K 2024-11-17T15:39:52,861 DEBUG [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/B/e40e83761dad438dac03051467d7c475 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/B/e40e83761dad438dac03051467d7c475 2024-11-17T15:39:52,863 INFO [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/B/e40e83761dad438dac03051467d7c475, entries=50, sequenceid=404, filesize=7.2 K 2024-11-17T15:39:52,864 DEBUG [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/.tmp/C/2eee554e2bb1407395c9fa199f0a54a8 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/C/2eee554e2bb1407395c9fa199f0a54a8 2024-11-17T15:39:52,866 INFO [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/C/2eee554e2bb1407395c9fa199f0a54a8, entries=50, sequenceid=404, filesize=7.2 K 2024-11-17T15:39:52,867 INFO [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegion(3140): Finished flush of dataSize ~13.42 KB/13740, heapSize ~35.86 KB/36720, currentSize=0 B/0 for a682ec5138f7084a1f2f9cd3afa94497 in 1645ms, sequenceid=404, compaction requested=true 2024-11-17T15:39:52,867 DEBUG [StoreCloser-TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/A/7f803d4371504753a4d9d9f9fd4d4306, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/A/9a38772ae8b349758f03f17c9a6030b4, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/A/c347b5c207114d43adf46db7ab4f0d75, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/A/0b280d5d14634e7c99536f38c498bc99, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/A/322cb0bcf1ed433eb0b62a26680ad739, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/A/30e1907f08e648819b7622b7594e1f17, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/A/2460528174d34b39bc887223334634b2, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/A/bc990395726143668c889819bd9e605c, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/A/fe806eb1dbd44354ba9f546e943ba3f7, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/A/3604334c372a467ab844d06f04087498, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/A/415ab553c1a54754be54ca8433b84166, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/A/dc16ad2d35744a4095376e009088dabc, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/A/29924c3a7b0a4a06ba910061ade6f630, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/A/aa4407ce8639450c9af5eed2e3d568f6, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/A/ebf05454530f40768b5a11e90355521b, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/A/2f81797602ce4532803ca5dced176683, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/A/876d0cd83fce43b6a195dc0ed4571d03, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/A/ec79ac6bdc6643f3a56d28cbceee8373, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/A/67a8fcefd62347529545281f4f125f5b, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/A/65e0522a067240c1bf0553ec1bf65235] to archive 2024-11-17T15:39:52,868 DEBUG [StoreCloser-TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-17T15:39:52,870 DEBUG [StoreCloser-TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/A/7f803d4371504753a4d9d9f9fd4d4306 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/A/7f803d4371504753a4d9d9f9fd4d4306 2024-11-17T15:39:52,871 DEBUG [StoreCloser-TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/A/9a38772ae8b349758f03f17c9a6030b4 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/A/9a38772ae8b349758f03f17c9a6030b4 2024-11-17T15:39:52,871 DEBUG [StoreCloser-TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/A/c347b5c207114d43adf46db7ab4f0d75 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/A/c347b5c207114d43adf46db7ab4f0d75 2024-11-17T15:39:52,872 DEBUG [StoreCloser-TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/A/0b280d5d14634e7c99536f38c498bc99 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/A/0b280d5d14634e7c99536f38c498bc99 2024-11-17T15:39:52,873 DEBUG [StoreCloser-TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/A/322cb0bcf1ed433eb0b62a26680ad739 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/A/322cb0bcf1ed433eb0b62a26680ad739 2024-11-17T15:39:52,874 DEBUG [StoreCloser-TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/A/30e1907f08e648819b7622b7594e1f17 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/A/30e1907f08e648819b7622b7594e1f17 2024-11-17T15:39:52,875 DEBUG [StoreCloser-TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/A/2460528174d34b39bc887223334634b2 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/A/2460528174d34b39bc887223334634b2 2024-11-17T15:39:52,875 DEBUG [StoreCloser-TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/A/bc990395726143668c889819bd9e605c to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/A/bc990395726143668c889819bd9e605c 2024-11-17T15:39:52,876 DEBUG [StoreCloser-TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/A/fe806eb1dbd44354ba9f546e943ba3f7 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/A/fe806eb1dbd44354ba9f546e943ba3f7 2024-11-17T15:39:52,877 DEBUG [StoreCloser-TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/A/3604334c372a467ab844d06f04087498 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/A/3604334c372a467ab844d06f04087498 2024-11-17T15:39:52,878 DEBUG [StoreCloser-TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/A/415ab553c1a54754be54ca8433b84166 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/A/415ab553c1a54754be54ca8433b84166 2024-11-17T15:39:52,878 DEBUG [StoreCloser-TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/A/dc16ad2d35744a4095376e009088dabc to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/A/dc16ad2d35744a4095376e009088dabc 2024-11-17T15:39:52,879 DEBUG [StoreCloser-TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/A/29924c3a7b0a4a06ba910061ade6f630 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/A/29924c3a7b0a4a06ba910061ade6f630 2024-11-17T15:39:52,880 DEBUG [StoreCloser-TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/A/aa4407ce8639450c9af5eed2e3d568f6 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/A/aa4407ce8639450c9af5eed2e3d568f6 2024-11-17T15:39:52,881 DEBUG [StoreCloser-TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/A/ebf05454530f40768b5a11e90355521b to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/A/ebf05454530f40768b5a11e90355521b 2024-11-17T15:39:52,881 DEBUG [StoreCloser-TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/A/2f81797602ce4532803ca5dced176683 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/A/2f81797602ce4532803ca5dced176683 2024-11-17T15:39:52,882 DEBUG [StoreCloser-TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/A/876d0cd83fce43b6a195dc0ed4571d03 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/A/876d0cd83fce43b6a195dc0ed4571d03 2024-11-17T15:39:52,883 DEBUG [StoreCloser-TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/A/ec79ac6bdc6643f3a56d28cbceee8373 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/A/ec79ac6bdc6643f3a56d28cbceee8373 2024-11-17T15:39:52,884 DEBUG [StoreCloser-TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/A/67a8fcefd62347529545281f4f125f5b to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/A/67a8fcefd62347529545281f4f125f5b 2024-11-17T15:39:52,885 DEBUG [StoreCloser-TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/A/65e0522a067240c1bf0553ec1bf65235 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/A/65e0522a067240c1bf0553ec1bf65235 2024-11-17T15:39:52,886 DEBUG [StoreCloser-TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/B/e6153b84a31b4f71a64111af4a821b0f, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/B/558ab7c6bb174c94a08e7cf77978faab, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/B/53559797764748e496cd4b4a2681c986, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/B/7b0f9b4518e5449dac6aa6827c7bb262, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/B/f9e1b0a87b6040bb963cd7ee8cd0aae9, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/B/16341dad9d3d415da4c870717af2a580, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/B/9bdb77f2b6614f818af43f71c5ea59ef, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/B/5f3f51f9e48a49d0bfadc9663cec248a, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/B/2f91faea19ac40edb92244a7bfec2b46, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/B/376b2eec641f4db18b5483d68ec80df1, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/B/edcccf8a10c041d28b11d339d20d70c4, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/B/9e065d0e15f44fec9daffa63c10ccc7d, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/B/3d3fc79339fe4c45972bc0f078d64545, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/B/57e89d6207814cf3858029798a607c9c, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/B/d2ce283fe14d4c3985e788acf6d96bdf, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/B/156f3d7037ae4101b961756214768686, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/B/08d112961f57431b8285e2d257fa1ace, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/B/c84029d3dbf04150874356bfbdc30517, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/B/acf9528f40ab418ca11bc1f10ced4af8, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/B/df4115e30f234f84a9eb1282e785e0fb] to archive 2024-11-17T15:39:52,887 DEBUG [StoreCloser-TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-17T15:39:52,888 DEBUG [StoreCloser-TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/B/e6153b84a31b4f71a64111af4a821b0f to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/B/e6153b84a31b4f71a64111af4a821b0f 2024-11-17T15:39:52,889 DEBUG [StoreCloser-TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/B/558ab7c6bb174c94a08e7cf77978faab to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/B/558ab7c6bb174c94a08e7cf77978faab 2024-11-17T15:39:52,890 DEBUG [StoreCloser-TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/B/53559797764748e496cd4b4a2681c986 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/B/53559797764748e496cd4b4a2681c986 2024-11-17T15:39:52,890 DEBUG [StoreCloser-TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/B/7b0f9b4518e5449dac6aa6827c7bb262 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/B/7b0f9b4518e5449dac6aa6827c7bb262 2024-11-17T15:39:52,891 DEBUG [StoreCloser-TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/B/f9e1b0a87b6040bb963cd7ee8cd0aae9 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/B/f9e1b0a87b6040bb963cd7ee8cd0aae9 2024-11-17T15:39:52,892 DEBUG [StoreCloser-TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/B/16341dad9d3d415da4c870717af2a580 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/B/16341dad9d3d415da4c870717af2a580 2024-11-17T15:39:52,893 DEBUG [StoreCloser-TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/B/9bdb77f2b6614f818af43f71c5ea59ef to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/B/9bdb77f2b6614f818af43f71c5ea59ef 2024-11-17T15:39:52,893 DEBUG [StoreCloser-TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/B/5f3f51f9e48a49d0bfadc9663cec248a to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/B/5f3f51f9e48a49d0bfadc9663cec248a 2024-11-17T15:39:52,894 DEBUG [StoreCloser-TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/B/2f91faea19ac40edb92244a7bfec2b46 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/B/2f91faea19ac40edb92244a7bfec2b46 2024-11-17T15:39:52,895 DEBUG [StoreCloser-TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/B/376b2eec641f4db18b5483d68ec80df1 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/B/376b2eec641f4db18b5483d68ec80df1 2024-11-17T15:39:52,896 DEBUG [StoreCloser-TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/B/edcccf8a10c041d28b11d339d20d70c4 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/B/edcccf8a10c041d28b11d339d20d70c4 2024-11-17T15:39:52,896 DEBUG [StoreCloser-TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/B/9e065d0e15f44fec9daffa63c10ccc7d to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/B/9e065d0e15f44fec9daffa63c10ccc7d 2024-11-17T15:39:52,897 DEBUG [StoreCloser-TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/B/3d3fc79339fe4c45972bc0f078d64545 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/B/3d3fc79339fe4c45972bc0f078d64545 2024-11-17T15:39:52,898 DEBUG [StoreCloser-TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/B/57e89d6207814cf3858029798a607c9c to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/B/57e89d6207814cf3858029798a607c9c 2024-11-17T15:39:52,899 DEBUG [StoreCloser-TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/B/d2ce283fe14d4c3985e788acf6d96bdf to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/B/d2ce283fe14d4c3985e788acf6d96bdf 2024-11-17T15:39:52,900 DEBUG [StoreCloser-TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/B/156f3d7037ae4101b961756214768686 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/B/156f3d7037ae4101b961756214768686 2024-11-17T15:39:52,900 DEBUG [StoreCloser-TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/B/08d112961f57431b8285e2d257fa1ace to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/B/08d112961f57431b8285e2d257fa1ace 2024-11-17T15:39:52,901 DEBUG [StoreCloser-TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/B/c84029d3dbf04150874356bfbdc30517 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/B/c84029d3dbf04150874356bfbdc30517 2024-11-17T15:39:52,902 DEBUG [StoreCloser-TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/B/acf9528f40ab418ca11bc1f10ced4af8 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/B/acf9528f40ab418ca11bc1f10ced4af8 2024-11-17T15:39:52,903 DEBUG [StoreCloser-TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/B/df4115e30f234f84a9eb1282e785e0fb to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/B/df4115e30f234f84a9eb1282e785e0fb 2024-11-17T15:39:52,904 DEBUG [StoreCloser-TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/C/e5e5127973af48a1b821b34f6fb3c536, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/C/64c61bc118674d349cf0ea1ad693fdc4, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/C/509b44c253af4543a1505011ae1c5251, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/C/b5f0237d91ac4bdb9d42b90f1f9ed19a, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/C/d15d0151925c461ba23a2bebd6fc4ed9, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/C/b10abbbc7aa1433daa4410a3d07f4769, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/C/72ba348337814dc690a03bfc5b238c08, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/C/cad4210b0b6d4656b6ec08af997136b5, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/C/690db289c6ff4559bf0f9ce78f77c41c, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/C/21ab8f1c786c40b6997dc6fa2a1cf6a4, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/C/cd225f855b884a6cbc022d4a0a571f55, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/C/2f12e08c211a4581a7d222a6bc31524d, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/C/0de620996731476fb19bcba0e55d3b17, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/C/c3ebde68060e41ce85ee6bb4cec4376f, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/C/738f2d5b6e084adcadf5c217b64796f6, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/C/b1f3f0b436ac4456a2ee333f58814330, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/C/135dff1cf42241348acb4c354a76866c, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/C/98f53ddc189e4c038be7d19425785d5d, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/C/34c973e46037411f8d07d43dbb49b98d, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/C/c7954d30fec540a885f670a7821e9690] to archive 2024-11-17T15:39:52,904 DEBUG [StoreCloser-TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-17T15:39:52,905 DEBUG [StoreCloser-TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/C/e5e5127973af48a1b821b34f6fb3c536 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/C/e5e5127973af48a1b821b34f6fb3c536 2024-11-17T15:39:52,906 DEBUG [StoreCloser-TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/C/64c61bc118674d349cf0ea1ad693fdc4 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/C/64c61bc118674d349cf0ea1ad693fdc4 2024-11-17T15:39:52,907 DEBUG [StoreCloser-TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/C/509b44c253af4543a1505011ae1c5251 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/C/509b44c253af4543a1505011ae1c5251 2024-11-17T15:39:52,908 DEBUG [StoreCloser-TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/C/b5f0237d91ac4bdb9d42b90f1f9ed19a to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/C/b5f0237d91ac4bdb9d42b90f1f9ed19a 2024-11-17T15:39:52,908 DEBUG [StoreCloser-TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/C/d15d0151925c461ba23a2bebd6fc4ed9 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/C/d15d0151925c461ba23a2bebd6fc4ed9 2024-11-17T15:39:52,909 DEBUG [StoreCloser-TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/C/b10abbbc7aa1433daa4410a3d07f4769 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/C/b10abbbc7aa1433daa4410a3d07f4769 2024-11-17T15:39:52,910 DEBUG [StoreCloser-TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/C/72ba348337814dc690a03bfc5b238c08 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/C/72ba348337814dc690a03bfc5b238c08 2024-11-17T15:39:52,910 DEBUG [StoreCloser-TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/C/cad4210b0b6d4656b6ec08af997136b5 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/C/cad4210b0b6d4656b6ec08af997136b5 2024-11-17T15:39:52,911 DEBUG [StoreCloser-TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/C/690db289c6ff4559bf0f9ce78f77c41c to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/C/690db289c6ff4559bf0f9ce78f77c41c 2024-11-17T15:39:52,912 DEBUG [StoreCloser-TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/C/21ab8f1c786c40b6997dc6fa2a1cf6a4 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/C/21ab8f1c786c40b6997dc6fa2a1cf6a4 2024-11-17T15:39:52,912 DEBUG [StoreCloser-TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/C/cd225f855b884a6cbc022d4a0a571f55 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/C/cd225f855b884a6cbc022d4a0a571f55 2024-11-17T15:39:52,913 DEBUG [StoreCloser-TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/C/2f12e08c211a4581a7d222a6bc31524d to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/C/2f12e08c211a4581a7d222a6bc31524d 2024-11-17T15:39:52,914 DEBUG [StoreCloser-TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/C/0de620996731476fb19bcba0e55d3b17 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/C/0de620996731476fb19bcba0e55d3b17 2024-11-17T15:39:52,914 DEBUG [StoreCloser-TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/C/c3ebde68060e41ce85ee6bb4cec4376f to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/C/c3ebde68060e41ce85ee6bb4cec4376f 2024-11-17T15:39:52,915 DEBUG [StoreCloser-TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/C/738f2d5b6e084adcadf5c217b64796f6 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/C/738f2d5b6e084adcadf5c217b64796f6 2024-11-17T15:39:52,916 DEBUG [StoreCloser-TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/C/b1f3f0b436ac4456a2ee333f58814330 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/C/b1f3f0b436ac4456a2ee333f58814330 2024-11-17T15:39:52,916 DEBUG [StoreCloser-TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/C/135dff1cf42241348acb4c354a76866c to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/C/135dff1cf42241348acb4c354a76866c 2024-11-17T15:39:52,917 DEBUG [StoreCloser-TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/C/98f53ddc189e4c038be7d19425785d5d to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/C/98f53ddc189e4c038be7d19425785d5d 2024-11-17T15:39:52,918 DEBUG [StoreCloser-TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/C/34c973e46037411f8d07d43dbb49b98d to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/C/34c973e46037411f8d07d43dbb49b98d 2024-11-17T15:39:52,918 DEBUG [StoreCloser-TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/C/c7954d30fec540a885f670a7821e9690 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/C/c7954d30fec540a885f670a7821e9690 2024-11-17T15:39:52,922 DEBUG [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/recovered.edits/407.seqid, newMaxSeqId=407, maxSeqId=4 2024-11-17T15:39:52,922 INFO [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497. 2024-11-17T15:39:52,922 DEBUG [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegion(1676): Region close journal for a682ec5138f7084a1f2f9cd3afa94497: Waiting for close lock at 1731857990627Running coprocessor pre-close hooks at 1731857990627Disabling compacts and flushes for region at 1731857990627Disabling writes for close at 1731857991222 (+595 ms)Obtaining lock to block concurrent updates at 1731857991222Preparing flush snapshotting stores in a682ec5138f7084a1f2f9cd3afa94497 at 1731857991222Finished memstore snapshotting TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497., syncing WAL and waiting on mvcc, flushsize=dataSize=13740, getHeapSize=36720, getOffHeapSize=0, getCellsCount=300 at 1731857991222Flushing stores of TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497. at 1731857991223 (+1 ms)Flushing a682ec5138f7084a1f2f9cd3afa94497/A: creating writer at 1731857991223Flushing mob file a682ec5138f7084a1f2f9cd3afa94497/A: appending metadata at 1731857991226 (+3 ms)Flushing mob file a682ec5138f7084a1f2f9cd3afa94497/A: closing flushed file at 1731857991226Flushing a682ec5138f7084a1f2f9cd3afa94497/A: appending metadata at 1731857991633 (+407 ms)Flushing a682ec5138f7084a1f2f9cd3afa94497/A: closing flushed file at 1731857991633Flushing a682ec5138f7084a1f2f9cd3afa94497/B: creating writer at 1731857992040 (+407 ms)Flushing a682ec5138f7084a1f2f9cd3afa94497/B: appending metadata at 1731857992042 (+2 ms)Flushing a682ec5138f7084a1f2f9cd3afa94497/B: closing flushed file at 1731857992042Flushing a682ec5138f7084a1f2f9cd3afa94497/C: creating writer at 1731857992449 (+407 ms)Flushing a682ec5138f7084a1f2f9cd3afa94497/C: appending metadata at 1731857992451 (+2 ms)Flushing a682ec5138f7084a1f2f9cd3afa94497/C: closing flushed file at 1731857992451Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@24922195: reopening flushed file at 1731857992857 (+406 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@545a4ee: reopening flushed file at 1731857992860 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2f1ab33d: reopening flushed file at 1731857992863 (+3 ms)Finished flush of dataSize ~13.42 KB/13740, heapSize ~35.86 KB/36720, currentSize=0 B/0 for a682ec5138f7084a1f2f9cd3afa94497 in 1645ms, sequenceid=404, compaction requested=true at 1731857992867 (+4 ms)Writing region close event to WAL at 1731857992919 (+52 ms)Running coprocessor post-close hooks at 1731857992922 (+3 ms)Closed at 1731857992922 2024-11-17T15:39:52,924 INFO [RS_CLOSE_REGION-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] handler.UnassignRegionHandler(157): Closed a682ec5138f7084a1f2f9cd3afa94497 2024-11-17T15:39:52,924 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=172 updating hbase:meta row=a682ec5138f7084a1f2f9cd3afa94497, regionState=CLOSED 2024-11-17T15:39:52,925 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=173, ppid=172, state=RUNNABLE, hasLock=false; CloseRegionProcedure a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 because future has completed 2024-11-17T15:39:52,927 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=173, resume processing ppid=172 2024-11-17T15:39:52,927 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=173, ppid=172, state=SUCCESS, hasLock=false; CloseRegionProcedure a682ec5138f7084a1f2f9cd3afa94497, server=7a780d55532c,32955,1731857800648 in 2.4520 sec 2024-11-17T15:39:52,929 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=172, resume processing ppid=171 2024-11-17T15:39:52,929 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=172, ppid=171, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a682ec5138f7084a1f2f9cd3afa94497, UNASSIGN in 2.4550 sec 2024-11-17T15:39:52,930 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=171, resume processing ppid=170 2024-11-17T15:39:52,930 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=171, ppid=170, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 2.4580 sec 2024-11-17T15:39:52,932 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731857992932"}]},"ts":"1731857992932"} 2024-11-17T15:39:52,933 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-17T15:39:52,933 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-17T15:39:52,935 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=170, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 2.4670 sec 2024-11-17T15:39:54,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-17T15:39:54,623 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-17T15:39:54,624 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-17T15:39:54,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] procedure2.ProcedureExecutor(1139): Stored pid=174, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T15:39:54,625 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=174, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T15:39:54,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-11-17T15:39:54,626 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=174, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T15:39:54,628 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497 2024-11-17T15:39:54,630 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/A, FileablePath, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/B, FileablePath, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/C, FileablePath, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/recovered.edits] 2024-11-17T15:39:54,632 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/A/347fe25c66c343b595fadd951715270a to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/A/347fe25c66c343b595fadd951715270a 2024-11-17T15:39:54,634 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/A/4d930bf6cfc741bfb1502ee70410e461 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/A/4d930bf6cfc741bfb1502ee70410e461 2024-11-17T15:39:54,634 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/A/6bef57857aaf41a9852641b7ff78ba0c to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/A/6bef57857aaf41a9852641b7ff78ba0c 2024-11-17T15:39:54,635 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/A/7d00fcbaffeb43c7a1d6887f9967e320 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/A/7d00fcbaffeb43c7a1d6887f9967e320 2024-11-17T15:39:54,636 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/A/d9d1ef44d1684eadb1b2105b5044e4ce to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/A/d9d1ef44d1684eadb1b2105b5044e4ce 2024-11-17T15:39:54,637 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/A/fedf2291c3734c7782107da4a45b4489 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/A/fedf2291c3734c7782107da4a45b4489 2024-11-17T15:39:54,638 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/B/5487658398544cc9b11a1ff9cb55a05f to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/B/5487658398544cc9b11a1ff9cb55a05f 2024-11-17T15:39:54,639 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/B/5bd1b00ecbb14a21bf905726c84bf16b to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/B/5bd1b00ecbb14a21bf905726c84bf16b 2024-11-17T15:39:54,640 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/B/a43eec3f55ec44c99e823369123d24dd to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/B/a43eec3f55ec44c99e823369123d24dd 2024-11-17T15:39:54,641 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/B/abfb95d6e1c94568aa9ad8260d173035 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/B/abfb95d6e1c94568aa9ad8260d173035 2024-11-17T15:39:54,642 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/B/e40e83761dad438dac03051467d7c475 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/B/e40e83761dad438dac03051467d7c475 2024-11-17T15:39:54,643 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/B/e4684a5ec56a4ba89b8c98266f3b0446 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/B/e4684a5ec56a4ba89b8c98266f3b0446 2024-11-17T15:39:54,644 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/C/16d78d7208684a24bf57057f37d67b32 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/C/16d78d7208684a24bf57057f37d67b32 2024-11-17T15:39:54,645 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/C/2eee554e2bb1407395c9fa199f0a54a8 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/C/2eee554e2bb1407395c9fa199f0a54a8 2024-11-17T15:39:54,646 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/C/4177325527e14e93a99e63d09d2fca17 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/C/4177325527e14e93a99e63d09d2fca17 2024-11-17T15:39:54,647 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/C/7eb3d058f5ce48ac9ce823525a16c2ed to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/C/7eb3d058f5ce48ac9ce823525a16c2ed 2024-11-17T15:39:54,648 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/C/9d5bc5340abf49c0a5e32f676c30e3d0 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/C/9d5bc5340abf49c0a5e32f676c30e3d0 2024-11-17T15:39:54,649 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/C/d2ac9f1b37ab4699bf8a9256f00e9c16 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/C/d2ac9f1b37ab4699bf8a9256f00e9c16 2024-11-17T15:39:54,651 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/recovered.edits/407.seqid to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497/recovered.edits/407.seqid 2024-11-17T15:39:54,651 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/default/TestAcidGuarantees/a682ec5138f7084a1f2f9cd3afa94497 2024-11-17T15:39:54,651 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-17T15:39:54,652 DEBUG [PEWorker-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-17T15:39:54,652 DEBUG [PEWorker-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-17T15:39:54,655 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117098e390e0d894b2b8a53f67c36bee6f2_a682ec5138f7084a1f2f9cd3afa94497 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117098e390e0d894b2b8a53f67c36bee6f2_a682ec5138f7084a1f2f9cd3afa94497 2024-11-17T15:39:54,656 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411170bf553063f0e40f2b4e1fd3cd042c0a7_a682ec5138f7084a1f2f9cd3afa94497 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411170bf553063f0e40f2b4e1fd3cd042c0a7_a682ec5138f7084a1f2f9cd3afa94497 2024-11-17T15:39:54,657 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411170ce47404dd9342a7a2e8483968c3d2a3_a682ec5138f7084a1f2f9cd3afa94497 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411170ce47404dd9342a7a2e8483968c3d2a3_a682ec5138f7084a1f2f9cd3afa94497 2024-11-17T15:39:54,658 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411170e6f8c330161497fb1295bcee41aa385_a682ec5138f7084a1f2f9cd3afa94497 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411170e6f8c330161497fb1295bcee41aa385_a682ec5138f7084a1f2f9cd3afa94497 2024-11-17T15:39:54,658 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117160e685a1a7846b482730209acdfbffb_a682ec5138f7084a1f2f9cd3afa94497 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117160e685a1a7846b482730209acdfbffb_a682ec5138f7084a1f2f9cd3afa94497 2024-11-17T15:39:54,659 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411171b14f80943c44cad88939a15a871990c_a682ec5138f7084a1f2f9cd3afa94497 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411171b14f80943c44cad88939a15a871990c_a682ec5138f7084a1f2f9cd3afa94497 2024-11-17T15:39:54,660 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411172580b345c34c442c95b6a4afe066eb1d_a682ec5138f7084a1f2f9cd3afa94497 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411172580b345c34c442c95b6a4afe066eb1d_a682ec5138f7084a1f2f9cd3afa94497 2024-11-17T15:39:54,661 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411172d9af8688bf44d039c379f7108218a0f_a682ec5138f7084a1f2f9cd3afa94497 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411172d9af8688bf44d039c379f7108218a0f_a682ec5138f7084a1f2f9cd3afa94497 2024-11-17T15:39:54,662 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117345e53bf303140c180594919a6fc5bd0_a682ec5138f7084a1f2f9cd3afa94497 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117345e53bf303140c180594919a6fc5bd0_a682ec5138f7084a1f2f9cd3afa94497 2024-11-17T15:39:54,663 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411175b0432fc31d14262a2f1f1e71f555c90_a682ec5138f7084a1f2f9cd3afa94497 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411175b0432fc31d14262a2f1f1e71f555c90_a682ec5138f7084a1f2f9cd3afa94497 2024-11-17T15:39:54,663 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111767a86801faef4ad7a212089a2f17ce7d_a682ec5138f7084a1f2f9cd3afa94497 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111767a86801faef4ad7a212089a2f17ce7d_a682ec5138f7084a1f2f9cd3afa94497 2024-11-17T15:39:54,664 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411176fce87b261414263be509354836f9610_a682ec5138f7084a1f2f9cd3afa94497 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411176fce87b261414263be509354836f9610_a682ec5138f7084a1f2f9cd3afa94497 2024-11-17T15:39:54,665 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411177fbed3a2d48d4655bac54573441b7fa9_a682ec5138f7084a1f2f9cd3afa94497 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411177fbed3a2d48d4655bac54573441b7fa9_a682ec5138f7084a1f2f9cd3afa94497 2024-11-17T15:39:54,666 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111798fbd0ace52f4f7cb0a62bb42d62ad69_a682ec5138f7084a1f2f9cd3afa94497 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111798fbd0ace52f4f7cb0a62bb42d62ad69_a682ec5138f7084a1f2f9cd3afa94497 2024-11-17T15:39:54,666 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111799daa4da7d0645419010868f434e2fc7_a682ec5138f7084a1f2f9cd3afa94497 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111799daa4da7d0645419010868f434e2fc7_a682ec5138f7084a1f2f9cd3afa94497 2024-11-17T15:39:54,667 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117bf5e6e4249894d9cba3301db3476ba0b_a682ec5138f7084a1f2f9cd3afa94497 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117bf5e6e4249894d9cba3301db3476ba0b_a682ec5138f7084a1f2f9cd3afa94497 2024-11-17T15:39:54,668 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117c98054f2dfde412580c0d9ae5ef198c4_a682ec5138f7084a1f2f9cd3afa94497 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117c98054f2dfde412580c0d9ae5ef198c4_a682ec5138f7084a1f2f9cd3afa94497 2024-11-17T15:39:54,669 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117cfb4d07d1c6b43a7b51c520d6f1aa66d_a682ec5138f7084a1f2f9cd3afa94497 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117cfb4d07d1c6b43a7b51c520d6f1aa66d_a682ec5138f7084a1f2f9cd3afa94497 2024-11-17T15:39:54,669 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117d3463b31c3ec47beae12dd5e54822790_a682ec5138f7084a1f2f9cd3afa94497 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117d3463b31c3ec47beae12dd5e54822790_a682ec5138f7084a1f2f9cd3afa94497 2024-11-17T15:39:54,670 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117da370652fdf347a496c9928e86863022_a682ec5138f7084a1f2f9cd3afa94497 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117da370652fdf347a496c9928e86863022_a682ec5138f7084a1f2f9cd3afa94497 2024-11-17T15:39:54,671 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117f0ee61d60418423d96979c68eb32a08b_a682ec5138f7084a1f2f9cd3afa94497 to hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117f0ee61d60418423d96979c68eb32a08b_a682ec5138f7084a1f2f9cd3afa94497 2024-11-17T15:39:54,671 DEBUG [PEWorker-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-17T15:39:54,673 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=174, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T15:39:54,675 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-17T15:39:54,677 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-17T15:39:54,678 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=174, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T15:39:54,678 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-17T15:39:54,678 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731857994678"}]},"ts":"9223372036854775807"} 2024-11-17T15:39:54,679 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-17T15:39:54,679 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => a682ec5138f7084a1f2f9cd3afa94497, NAME => 'TestAcidGuarantees,,1731857965733.a682ec5138f7084a1f2f9cd3afa94497.', STARTKEY => '', ENDKEY => ''}] 2024-11-17T15:39:54,679 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-17T15:39:54,679 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731857994679"}]},"ts":"9223372036854775807"} 2024-11-17T15:39:54,681 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-17T15:39:54,681 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=174, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T15:39:54,682 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=174, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 57 msec 2024-11-17T15:39:54,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-11-17T15:39:54,733 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-17T15:39:54,733 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-17T15:39:54,745 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobGetAtomicity Thread=242 (was 237) - Thread LEAK? -, OpenFileDescriptor=457 (was 443) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=427 (was 389) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=2680 (was 2744) 2024-11-17T15:39:54,745 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-17T15:39:54,745 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-17T15:39:54,745 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-17T15:39:54,745 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:39:54,745 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:39:54,745 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T15:39:54,745 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-17T15:39:54,745 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=129225742, stopped=false 2024-11-17T15:39:54,746 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=7a780d55532c,44223,1731857799978 2024-11-17T15:39:54,749 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32955-0x1012692c2820001, quorum=127.0.0.1:59618, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-17T15:39:54,749 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44223-0x1012692c2820000, quorum=127.0.0.1:59618, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-17T15:39:54,749 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32955-0x1012692c2820001, quorum=127.0.0.1:59618, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T15:39:54,749 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44223-0x1012692c2820000, quorum=127.0.0.1:59618, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T15:39:54,749 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-17T15:39:54,750 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-17T15:39:54,750 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:32955-0x1012692c2820001, quorum=127.0.0.1:59618, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-17T15:39:54,750 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:44223-0x1012692c2820000, quorum=127.0.0.1:59618, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-17T15:39:54,750 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-17T15:39:54,751 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:39:54,751 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '7a780d55532c,32955,1731857800648' ***** 2024-11-17T15:39:54,751 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-17T15:39:54,751 INFO [RS:0;7a780d55532c:32955 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-17T15:39:54,752 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-17T15:39:54,752 INFO [RS:0;7a780d55532c:32955 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-17T15:39:54,752 INFO [RS:0;7a780d55532c:32955 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-17T15:39:54,752 INFO [RS:0;7a780d55532c:32955 {}] regionserver.HRegionServer(959): stopping server 7a780d55532c,32955,1731857800648 2024-11-17T15:39:54,752 INFO [RS:0;7a780d55532c:32955 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-17T15:39:54,752 INFO [RS:0;7a780d55532c:32955 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;7a780d55532c:32955. 2024-11-17T15:39:54,752 DEBUG [RS:0;7a780d55532c:32955 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-17T15:39:54,752 DEBUG [RS:0;7a780d55532c:32955 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:39:54,752 INFO [RS:0;7a780d55532c:32955 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-17T15:39:54,752 INFO [RS:0;7a780d55532c:32955 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-17T15:39:54,752 INFO [RS:0;7a780d55532c:32955 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-17T15:39:54,752 INFO [RS:0;7a780d55532c:32955 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-17T15:39:54,753 INFO [RS:0;7a780d55532c:32955 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-17T15:39:54,753 DEBUG [RS:0;7a780d55532c:32955 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-17T15:39:54,753 DEBUG [RS:0;7a780d55532c:32955 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-17T15:39:54,753 DEBUG [RS_CLOSE_META-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-17T15:39:54,754 INFO [RS_CLOSE_META-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-17T15:39:54,754 DEBUG [RS_CLOSE_META-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-17T15:39:54,754 DEBUG [RS_CLOSE_META-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-17T15:39:54,754 DEBUG [RS_CLOSE_META-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-17T15:39:54,754 INFO [RS_CLOSE_META-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-11-17T15:39:54,778 DEBUG [RS_CLOSE_META-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/hbase/meta/1588230740/.tmp/info/5033375fc766405a853ad1516826129c is 95, key is TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3./info:/1731857832724/DeleteFamily/seqid=0 2024-11-17T15:39:54,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742458_1634 (size=6514) 2024-11-17T15:39:54,798 INFO [regionserver/7a780d55532c:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-17T15:39:54,953 DEBUG [RS:0;7a780d55532c:32955 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-17T15:39:55,154 DEBUG [RS:0;7a780d55532c:32955 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-17T15:39:55,182 INFO [RS_CLOSE_META-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/hbase/meta/1588230740/.tmp/info/5033375fc766405a853ad1516826129c 2024-11-17T15:39:55,202 DEBUG [RS_CLOSE_META-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/hbase/meta/1588230740/.tmp/ns/fe391616dbc547609fc0a0a625dd41b6 is 93, key is TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3./ns:/1731857832680/DeleteFamily/seqid=0 2024-11-17T15:39:55,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742459_1635 (size=5906) 2024-11-17T15:39:55,215 INFO [regionserver/7a780d55532c:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-17T15:39:55,215 INFO [regionserver/7a780d55532c:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-17T15:39:55,354 DEBUG [RS:0;7a780d55532c:32955 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-17T15:39:55,554 DEBUG [RS:0;7a780d55532c:32955 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-17T15:39:55,607 INFO [RS_CLOSE_META-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/hbase/meta/1588230740/.tmp/ns/fe391616dbc547609fc0a0a625dd41b6 2024-11-17T15:39:55,625 DEBUG [RS_CLOSE_META-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/hbase/meta/1588230740/.tmp/rep_barrier/c868952df1d240a397d2a160d649cbab is 102, key is TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3./rep_barrier:/1731857832680/DeleteFamily/seqid=0 2024-11-17T15:39:55,628 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742460_1636 (size=6025) 2024-11-17T15:39:55,754 DEBUG [RS:0;7a780d55532c:32955 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-17T15:39:55,954 DEBUG [RS:0;7a780d55532c:32955 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-17T15:39:56,029 INFO [RS_CLOSE_META-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/hbase/meta/1588230740/.tmp/rep_barrier/c868952df1d240a397d2a160d649cbab 2024-11-17T15:39:56,046 DEBUG [RS_CLOSE_META-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/hbase/meta/1588230740/.tmp/table/389b2b737cdf474eb487166f59462229 is 96, key is TestAcidGuarantees,,1731857804760.a80c4e623aab44bea3cfa33ee4fee8f3./table:/1731857832680/DeleteFamily/seqid=0 2024-11-17T15:39:56,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742461_1637 (size=5936) 2024-11-17T15:39:56,155 DEBUG [RS:0;7a780d55532c:32955 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-17T15:39:56,355 DEBUG [RS:0;7a780d55532c:32955 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-17T15:39:56,450 INFO [RS_CLOSE_META-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/hbase/meta/1588230740/.tmp/table/389b2b737cdf474eb487166f59462229 2024-11-17T15:39:56,453 DEBUG [RS_CLOSE_META-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/hbase/meta/1588230740/.tmp/info/5033375fc766405a853ad1516826129c as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/hbase/meta/1588230740/info/5033375fc766405a853ad1516826129c 2024-11-17T15:39:56,455 INFO [RS_CLOSE_META-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/hbase/meta/1588230740/info/5033375fc766405a853ad1516826129c, entries=12, sequenceid=90, filesize=6.4 K 2024-11-17T15:39:56,456 DEBUG [RS_CLOSE_META-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/hbase/meta/1588230740/.tmp/ns/fe391616dbc547609fc0a0a625dd41b6 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/hbase/meta/1588230740/ns/fe391616dbc547609fc0a0a625dd41b6 2024-11-17T15:39:56,458 INFO [RS_CLOSE_META-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/hbase/meta/1588230740/ns/fe391616dbc547609fc0a0a625dd41b6, entries=8, sequenceid=90, filesize=5.8 K 2024-11-17T15:39:56,459 DEBUG [RS_CLOSE_META-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/hbase/meta/1588230740/.tmp/rep_barrier/c868952df1d240a397d2a160d649cbab as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/hbase/meta/1588230740/rep_barrier/c868952df1d240a397d2a160d649cbab 2024-11-17T15:39:56,461 INFO [RS_CLOSE_META-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/hbase/meta/1588230740/rep_barrier/c868952df1d240a397d2a160d649cbab, entries=6, sequenceid=90, filesize=5.9 K 2024-11-17T15:39:56,461 DEBUG [RS_CLOSE_META-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/hbase/meta/1588230740/.tmp/table/389b2b737cdf474eb487166f59462229 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/hbase/meta/1588230740/table/389b2b737cdf474eb487166f59462229 2024-11-17T15:39:56,463 INFO [RS_CLOSE_META-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/hbase/meta/1588230740/table/389b2b737cdf474eb487166f59462229, entries=7, sequenceid=90, filesize=5.8 K 2024-11-17T15:39:56,464 INFO [RS_CLOSE_META-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1710ms, sequenceid=90, compaction requested=false 2024-11-17T15:39:56,467 DEBUG [RS_CLOSE_META-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-11-17T15:39:56,468 DEBUG [RS_CLOSE_META-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-17T15:39:56,468 INFO [RS_CLOSE_META-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-17T15:39:56,468 DEBUG [RS_CLOSE_META-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731857994753Running coprocessor pre-close hooks at 1731857994753Disabling compacts and flushes for region at 1731857994753Disabling writes for close at 1731857994754 (+1 ms)Obtaining lock to block concurrent updates at 1731857994754Preparing flush snapshotting stores in 1588230740 at 1731857994754Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1731857994754Flushing stores of hbase:meta,,1.1588230740 at 1731857994754Flushing 1588230740/info: creating writer at 1731857994755 (+1 ms)Flushing 1588230740/info: appending metadata at 1731857994778 (+23 ms)Flushing 1588230740/info: closing flushed file at 1731857994778Flushing 1588230740/ns: creating writer at 1731857995185 (+407 ms)Flushing 1588230740/ns: appending metadata at 1731857995201 (+16 ms)Flushing 1588230740/ns: closing flushed file at 1731857995201Flushing 1588230740/rep_barrier: creating writer at 1731857995610 (+409 ms)Flushing 1588230740/rep_barrier: appending metadata at 1731857995624 (+14 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1731857995624Flushing 1588230740/table: creating writer at 1731857996031 (+407 ms)Flushing 1588230740/table: appending metadata at 1731857996046 (+15 ms)Flushing 1588230740/table: closing flushed file at 1731857996046Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@63e3842d: reopening flushed file at 1731857996452 (+406 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@537a923e: reopening flushed file at 1731857996456 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@17876446: reopening flushed file at 1731857996458 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@70aab8c0: reopening flushed file at 1731857996461 (+3 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1710ms, sequenceid=90, compaction requested=false at 1731857996464 (+3 ms)Writing region close event to WAL at 1731857996465 (+1 ms)Running coprocessor post-close hooks at 1731857996468 (+3 ms)Closed at 1731857996468 2024-11-17T15:39:56,468 DEBUG [RS_CLOSE_META-regionserver/7a780d55532c:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-17T15:39:56,555 INFO [RS:0;7a780d55532c:32955 {}] regionserver.HRegionServer(976): stopping server 7a780d55532c,32955,1731857800648; all regions closed. 2024-11-17T15:39:56,559 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073741834_1010 (size=25344) 2024-11-17T15:39:56,561 DEBUG [RS:0;7a780d55532c:32955 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/oldWALs 2024-11-17T15:39:56,561 INFO [RS:0;7a780d55532c:32955 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 7a780d55532c%2C32955%2C1731857800648.meta:.meta(num 1731857804283) 2024-11-17T15:39:56,563 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073741832_1008 (size=14411730) 2024-11-17T15:39:56,564 DEBUG [RS:0;7a780d55532c:32955 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/oldWALs 2024-11-17T15:39:56,564 INFO [RS:0;7a780d55532c:32955 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 7a780d55532c%2C32955%2C1731857800648:(num 1731857803343) 2024-11-17T15:39:56,564 DEBUG [RS:0;7a780d55532c:32955 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T15:39:56,564 INFO [RS:0;7a780d55532c:32955 {}] regionserver.LeaseManager(133): Closed leases 2024-11-17T15:39:56,564 INFO [RS:0;7a780d55532c:32955 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-17T15:39:56,565 INFO [RS:0;7a780d55532c:32955 {}] hbase.ChoreService(370): Chore service for: regionserver/7a780d55532c:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-17T15:39:56,565 INFO [RS:0;7a780d55532c:32955 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-17T15:39:56,565 INFO [regionserver/7a780d55532c:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-17T15:39:56,565 INFO [RS:0;7a780d55532c:32955 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:32955 2024-11-17T15:39:56,569 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32955-0x1012692c2820001, quorum=127.0.0.1:59618, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/7a780d55532c,32955,1731857800648 2024-11-17T15:39:56,569 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44223-0x1012692c2820000, quorum=127.0.0.1:59618, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-17T15:39:56,569 INFO [RS:0;7a780d55532c:32955 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-17T15:39:56,571 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [7a780d55532c,32955,1731857800648] 2024-11-17T15:39:56,572 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/7a780d55532c,32955,1731857800648 already deleted, retry=false 2024-11-17T15:39:56,573 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 7a780d55532c,32955,1731857800648 expired; onlineServers=0 2024-11-17T15:39:56,573 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '7a780d55532c,44223,1731857799978' ***** 2024-11-17T15:39:56,573 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-17T15:39:56,573 INFO [M:0;7a780d55532c:44223 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-17T15:39:56,573 INFO [M:0;7a780d55532c:44223 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-17T15:39:56,573 DEBUG [M:0;7a780d55532c:44223 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-17T15:39:56,573 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-17T15:39:56,573 DEBUG [M:0;7a780d55532c:44223 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-17T15:39:56,573 DEBUG [master/7a780d55532c:0:becomeActiveMaster-HFileCleaner.small.0-1731857803009 {}] cleaner.HFileCleaner(306): Exit Thread[master/7a780d55532c:0:becomeActiveMaster-HFileCleaner.small.0-1731857803009,5,FailOnTimeoutGroup] 2024-11-17T15:39:56,573 DEBUG [master/7a780d55532c:0:becomeActiveMaster-HFileCleaner.large.0-1731857803002 {}] cleaner.HFileCleaner(306): Exit Thread[master/7a780d55532c:0:becomeActiveMaster-HFileCleaner.large.0-1731857803002,5,FailOnTimeoutGroup] 2024-11-17T15:39:56,573 INFO [M:0;7a780d55532c:44223 {}] hbase.ChoreService(370): Chore service for: master/7a780d55532c:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-17T15:39:56,573 INFO [M:0;7a780d55532c:44223 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-17T15:39:56,574 DEBUG [M:0;7a780d55532c:44223 {}] master.HMaster(1795): Stopping service threads 2024-11-17T15:39:56,574 INFO [M:0;7a780d55532c:44223 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-17T15:39:56,574 INFO [M:0;7a780d55532c:44223 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-17T15:39:56,574 ERROR [M:0;7a780d55532c:44223 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[IPC Client (1102635514) connection to localhost/127.0.0.1:39013 from jenkins,5,PEWorkerGroup] Thread[IPC Parameter Sending Thread for localhost/127.0.0.1:39013,5,PEWorkerGroup] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-11-17T15:39:56,574 INFO [M:0;7a780d55532c:44223 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-17T15:39:56,574 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-17T15:39:56,576 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44223-0x1012692c2820000, quorum=127.0.0.1:59618, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-17T15:39:56,576 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44223-0x1012692c2820000, quorum=127.0.0.1:59618, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T15:39:56,576 DEBUG [M:0;7a780d55532c:44223 {}] zookeeper.ZKUtil(347): master:44223-0x1012692c2820000, quorum=127.0.0.1:59618, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-17T15:39:56,576 WARN [M:0;7a780d55532c:44223 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-17T15:39:56,577 INFO [M:0;7a780d55532c:44223 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/.lastflushedseqids 2024-11-17T15:39:56,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742462_1638 (size=72) 2024-11-17T15:39:56,672 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32955-0x1012692c2820001, quorum=127.0.0.1:59618, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-17T15:39:56,672 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32955-0x1012692c2820001, quorum=127.0.0.1:59618, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-17T15:39:56,672 INFO [RS:0;7a780d55532c:32955 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-17T15:39:56,672 INFO [RS:0;7a780d55532c:32955 {}] regionserver.HRegionServer(1031): Exiting; stopping=7a780d55532c,32955,1731857800648; zookeeper connection closed. 2024-11-17T15:39:56,673 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@127fc76c {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@127fc76c 2024-11-17T15:39:56,673 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-17T15:39:56,986 INFO [M:0;7a780d55532c:44223 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-17T15:39:56,987 INFO [M:0;7a780d55532c:44223 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-17T15:39:56,987 DEBUG [M:0;7a780d55532c:44223 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-17T15:39:56,987 INFO [M:0;7a780d55532c:44223 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-17T15:39:56,987 DEBUG [M:0;7a780d55532c:44223 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-17T15:39:56,987 DEBUG [M:0;7a780d55532c:44223 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-17T15:39:56,987 DEBUG [M:0;7a780d55532c:44223 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-17T15:39:56,987 INFO [M:0;7a780d55532c:44223 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=749.85 KB heapSize=921.02 KB 2024-11-17T15:39:57,003 DEBUG [M:0;7a780d55532c:44223 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/b6170fd718a1451a88b4fd8da5589da8 is 82, key is hbase:meta,,1/info:regioninfo/1731857804398/Put/seqid=0 2024-11-17T15:39:57,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742463_1639 (size=5672) 2024-11-17T15:39:57,406 INFO [M:0;7a780d55532c:44223 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2105 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/b6170fd718a1451a88b4fd8da5589da8 2024-11-17T15:39:57,426 DEBUG [M:0;7a780d55532c:44223 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/118d1f3c431742fb83c0772602de3262 is 2277, key is \x00\x00\x00\x00\x00\x00\x00\x91/proc:d/1731857968799/Put/seqid=0 2024-11-17T15:39:57,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742464_1640 (size=44897) 2024-11-17T15:39:57,831 INFO [M:0;7a780d55532c:44223 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=749.29 KB at sequenceid=2105 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/118d1f3c431742fb83c0772602de3262 2024-11-17T15:39:57,835 INFO [M:0;7a780d55532c:44223 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 118d1f3c431742fb83c0772602de3262 2024-11-17T15:39:57,856 DEBUG [M:0;7a780d55532c:44223 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/830315b14154451995be52914fcabb3c is 69, key is 7a780d55532c,32955,1731857800648/rs:state/1731857803104/Put/seqid=0 2024-11-17T15:39:57,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073742465_1641 (size=5156) 2024-11-17T15:39:58,263 INFO [M:0;7a780d55532c:44223 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2105 (bloomFilter=true), to=hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/830315b14154451995be52914fcabb3c 2024-11-17T15:39:58,267 DEBUG [M:0;7a780d55532c:44223 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/b6170fd718a1451a88b4fd8da5589da8 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/b6170fd718a1451a88b4fd8da5589da8 2024-11-17T15:39:58,270 INFO [M:0;7a780d55532c:44223 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/b6170fd718a1451a88b4fd8da5589da8, entries=8, sequenceid=2105, filesize=5.5 K 2024-11-17T15:39:58,270 DEBUG [M:0;7a780d55532c:44223 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/118d1f3c431742fb83c0772602de3262 as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/118d1f3c431742fb83c0772602de3262 2024-11-17T15:39:58,273 INFO [M:0;7a780d55532c:44223 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 118d1f3c431742fb83c0772602de3262 2024-11-17T15:39:58,273 INFO [M:0;7a780d55532c:44223 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/118d1f3c431742fb83c0772602de3262, entries=174, sequenceid=2105, filesize=43.8 K 2024-11-17T15:39:58,273 DEBUG [M:0;7a780d55532c:44223 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/830315b14154451995be52914fcabb3c as hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/830315b14154451995be52914fcabb3c 2024-11-17T15:39:58,276 INFO [M:0;7a780d55532c:44223 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39013/user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/830315b14154451995be52914fcabb3c, entries=1, sequenceid=2105, filesize=5.0 K 2024-11-17T15:39:58,277 INFO [M:0;7a780d55532c:44223 {}] regionserver.HRegion(3140): Finished flush of dataSize ~749.85 KB/767842, heapSize ~920.73 KB/942824, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1290ms, sequenceid=2105, compaction requested=false 2024-11-17T15:39:58,278 INFO [M:0;7a780d55532c:44223 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-17T15:39:58,278 DEBUG [M:0;7a780d55532c:44223 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731857996987Disabling compacts and flushes for region at 1731857996987Disabling writes for close at 1731857996987Obtaining lock to block concurrent updates at 1731857996987Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731857996987Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=767842, getHeapSize=943064, getOffHeapSize=0, getCellsCount=2304 at 1731857996987Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731857996988 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731857996988Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731857997002 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731857997002Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731857997409 (+407 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731857997426 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731857997426Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731857997835 (+409 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731857997856 (+21 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731857997856Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7049f414: reopening flushed file at 1731857998266 (+410 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7d7269c2: reopening flushed file at 1731857998270 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@73ceced3: reopening flushed file at 1731857998273 (+3 ms)Finished flush of dataSize ~749.85 KB/767842, heapSize ~920.73 KB/942824, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1290ms, sequenceid=2105, compaction requested=false at 1731857998277 (+4 ms)Writing region close event to WAL at 1731857998278 (+1 ms)Closed at 1731857998278 2024-11-17T15:39:58,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46871 is added to blk_1073741830_1006 (size=905882) 2024-11-17T15:39:58,281 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/c6d23c13-be9b-f906-3695-981df5de1206/MasterData/WALs/7a780d55532c,44223,1731857799978/7a780d55532c%2C44223%2C1731857799978.1731857802357 not finished, retry = 0 2024-11-17T15:39:58,382 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-17T15:39:58,382 INFO [M:0;7a780d55532c:44223 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-17T15:39:58,382 INFO [M:0;7a780d55532c:44223 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:44223 2024-11-17T15:39:58,382 INFO [M:0;7a780d55532c:44223 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-17T15:39:58,438 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-17T15:39:58,484 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44223-0x1012692c2820000, quorum=127.0.0.1:59618, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-17T15:39:58,484 INFO [M:0;7a780d55532c:44223 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-17T15:39:58,484 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44223-0x1012692c2820000, quorum=127.0.0.1:59618, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-17T15:39:58,489 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4269bd31{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-17T15:39:58,491 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-17T15:39:58,491 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-17T15:39:58,491 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-17T15:39:58,491 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6ce30f36-a42e-0287-cfce-b614a9c6228b/hadoop.log.dir/,STOPPED} 2024-11-17T15:39:58,494 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-17T15:39:58,494 WARN [BP-976341801-172.17.0.2-1731857797236 heartbeating to localhost/127.0.0.1:39013 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-17T15:39:58,494 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-17T15:39:58,494 WARN [BP-976341801-172.17.0.2-1731857797236 heartbeating to localhost/127.0.0.1:39013 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-976341801-172.17.0.2-1731857797236 (Datanode Uuid d17f3f9e-828e-4c49-97dc-417781b49415) service to localhost/127.0.0.1:39013 2024-11-17T15:39:58,496 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6ce30f36-a42e-0287-cfce-b614a9c6228b/cluster_3bb18fcf-0ee3-9799-c677-a8d421ff97cf/data/data1/current/BP-976341801-172.17.0.2-1731857797236 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-17T15:39:58,496 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6ce30f36-a42e-0287-cfce-b614a9c6228b/cluster_3bb18fcf-0ee3-9799-c677-a8d421ff97cf/data/data2/current/BP-976341801-172.17.0.2-1731857797236 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-17T15:39:58,497 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-17T15:39:58,503 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4ded99d3{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-17T15:39:58,504 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-17T15:39:58,504 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-17T15:39:58,504 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-17T15:39:58,504 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6ce30f36-a42e-0287-cfce-b614a9c6228b/hadoop.log.dir/,STOPPED} 2024-11-17T15:39:58,520 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-17T15:39:58,638 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down